@hugesb/dsh-stream-agent 0.0.0-stage → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. package/CHANGELOG.md +622 -0
  2. package/CONTRIBUTING.md +72 -0
  3. package/LICENSE +202 -0
  4. package/README.en.md +349 -0
  5. package/README.md +300 -3
  6. package/cordis.patch.yml +77 -0
  7. package/dist/entry/action-gate.js +206 -0
  8. package/dist/entry/adapter-kit.js +192 -0
  9. package/dist/entry/context-render.js +119 -0
  10. package/dist/entry/control-bar-cli.js +289 -0
  11. package/dist/entry/control-bar.js +123 -0
  12. package/dist/entry/control-session.js +115 -0
  13. package/dist/entry/control-state.js +125 -0
  14. package/dist/entry/cordis-compat.js +95 -0
  15. package/dist/entry/eval-probe.js +112 -0
  16. package/dist/entry/intent-log.js +121 -0
  17. package/dist/entry/json-persistence.js +306 -0
  18. package/dist/entry/loop.js +4163 -0
  19. package/dist/entry/mock-provider.js +549 -0
  20. package/dist/entry/obs-broker.js +178 -0
  21. package/dist/entry/ptc-bridge.js +405 -0
  22. package/dist/entry/state-handle.js +217 -0
  23. package/dist/entry/takeover-registry.js +60 -0
  24. package/dist/kernel/control-bar.js +559 -0
  25. package/dist/kernel/final.js +62 -0
  26. package/dist/kernel/gate.js +464 -0
  27. package/dist/kernel/modes.js +253 -0
  28. package/dist/kernel/obs.js +517 -0
  29. package/dist/kernel/parser.js +1081 -0
  30. package/dist/kernel/render.js +466 -0
  31. package/dist/kernel/state.js +416 -0
  32. package/dist/version.js +9 -0
  33. package/docs/TEST-PROPOSITIONS.md +139 -0
  34. package/docs/adr-001-segment-boundary.md +61 -0
  35. package/docs/architecture.md +104 -0
  36. package/docs/desktop-ui-feasibility.md +277 -0
  37. package/docs/disabled-lines-audit.md +53 -0
  38. package/docs/limitations.md +181 -0
  39. package/docs/protocol.md +99 -0
  40. package/docs/v0.4.0-plan.md +50 -0
  41. package/eval/METRICS.md +128 -0
  42. package/eval/RESULTS.md +246 -0
  43. package/eval/tasks/fixtures/big-report.md +298 -0
  44. package/eval/tasks/fixtures/build-manifest.txt +5 -0
  45. package/eval/tasks/fixtures/deploy.yaml +4 -0
  46. package/eval/tasks/fixtures/error.log +200 -0
  47. package/eval/tasks/fixtures/features.yaml +4 -0
  48. package/eval/tasks/fixtures/pipeline.yaml +4 -0
  49. package/eval/tasks/fixtures/sales-q1.csv +61 -0
  50. package/eval/tasks/fixtures/settings.yaml +4 -0
  51. package/eval/tasks/fixtures/users-roster.json +304 -0
  52. package/eval/tasks/fixtures/users-scores.json +20 -0
  53. package/eval/tasks/gen-fixtures.cjs +80 -0
  54. package/eval/tasks/tasks.json +150 -0
  55. package/eval/tools/f6-kill-drill.cjs +176 -0
  56. package/eval/tools/f6-kill-drill.worker.cjs +131 -0
  57. package/eval/tools/gen-results.cjs +191 -0
  58. package/eval/tools/machine-test.cjs +200 -0
  59. package/eval/tools/manual-code-scenarios.md +59 -0
  60. package/eval/tools/replay-check.cjs +311 -0
  61. package/eval/tools/runner.cjs +219 -0
  62. package/eval/tools/session-log.cjs +106 -0
  63. package/eval/tools/summarize.cjs +185 -0
  64. package/eval/tools/t3-cells.cjs +173 -0
  65. package/package.json +73 -6
  66. package/patches/adapters-smoke.yml +41 -0
  67. package/patches/eval-account.yml +36 -0
  68. package/patches/eval-baseline.yml +33 -0
  69. package/patches/eval-stream-code-off.yml +46 -0
  70. package/patches/eval-stream-code-on.yml +47 -0
  71. package/patches/eval-stream.yml +54 -0
  72. package/patches/takeover.yml +42 -0
  73. package/patches/test-preset.yml +23 -0
  74. package/preset/stream/cordis.patch.yml +39 -0
  75. package/preset/stream/package.json +13 -0
  76. package/preset.yml +56 -0
  77. package/profiles/stream/README.md +66 -0
  78. package/profiles/stream/cordis.patch.yml +12 -0
  79. package/profiles/stream/package.json +17 -0
  80. package/profiles/stream-headless/package.json +16 -0
  81. package/scripts/build.mjs +103 -0
  82. package/scripts/check-consistency.mjs +146 -0
  83. package/scripts/gate.mjs +138 -0
  84. package/scripts/install-check.mjs +116 -0
  85. package/scripts/install-cycle.mjs +88 -0
  86. package/scripts/release.mjs +428 -0
  87. package/scripts/repair-session-logs.mjs +498 -0
  88. package/src/entry/action-gate.ts +206 -0
  89. package/src/entry/adapter-kit.ts +192 -0
  90. package/src/entry/context-render.ts +119 -0
  91. package/src/entry/control-bar-cli.ts +289 -0
  92. package/src/entry/control-bar.ts +123 -0
  93. package/src/entry/control-session.ts +115 -0
  94. package/src/entry/control-state.ts +125 -0
  95. package/src/entry/cordis-compat.ts +95 -0
  96. package/src/entry/eval-probe.ts +112 -0
  97. package/src/entry/intent-log.ts +121 -0
  98. package/src/entry/json-persistence.ts +306 -0
  99. package/src/entry/loop.ts +4163 -0
  100. package/src/entry/mock-provider.ts +549 -0
  101. package/src/entry/obs-broker.ts +178 -0
  102. package/src/entry/ptc-bridge.ts +405 -0
  103. package/src/entry/state-handle.ts +217 -0
  104. package/src/entry/takeover-registry.ts +60 -0
  105. package/src/kernel/control-bar.ts +559 -0
  106. package/src/kernel/final.ts +62 -0
  107. package/src/kernel/gate.ts +464 -0
  108. package/src/kernel/modes.ts +253 -0
  109. package/src/kernel/obs.ts +517 -0
  110. package/src/kernel/parser.ts +1081 -0
  111. package/src/kernel/render.ts +466 -0
  112. package/src/kernel/state.ts +416 -0
  113. package/src/version.ts +9 -0
  114. package/test/activation.test.ts +187 -0
  115. package/test/adapters.test.ts +676 -0
  116. package/test/control-bar.test.ts +486 -0
  117. package/test/desktop-ui.test.ts +359 -0
  118. package/test/eval-probe.test.ts +139 -0
  119. package/test/f5-plan.test.ts +221 -0
  120. package/test/f6-intent.test.ts +168 -0
  121. package/test/f7-segment.test.ts +190 -0
  122. package/test/fix-limits.test.ts +599 -0
  123. package/test/gate.test.ts +380 -0
  124. package/test/helpers/stub-loop.ts +240 -0
  125. package/test/host-conformance.test.ts +418 -0
  126. package/test/kernel/obs.test.ts +329 -0
  127. package/test/kernel/parser.test.ts +451 -0
  128. package/test/loop.test.ts +1393 -0
  129. package/test/mock-provider.test.ts +386 -0
  130. package/test/ptc-bridge.test.ts +356 -0
  131. package/test/render.test.ts +418 -0
  132. package/test/s2-install-check.test.ts +46 -0
  133. package/test/scenarios/3-file-basic.json +13 -0
  134. package/test/scenarios/5-file-basic.json +22 -0
  135. package/test/scenarios/burn-more.json +9 -0
  136. package/test/scenarios/degraded-final.json +10 -0
  137. package/test/scenarios/fault-abort-xml.json +11 -0
  138. package/test/scenarios/fault-malformed-command.json +12 -0
  139. package/test/scenarios/fault-midstream-abort.json +11 -0
  140. package/test/scenarios/fault-saturating-output.json +9 -0
  141. package/test/scenarios/fault-timeout.json +9 -0
  142. package/test/scenarios/peek-first.json +11 -0
  143. package/test/scenarios/peek-second.json +11 -0
  144. package/test/scenarios/peek-third.json +10 -0
  145. package/test/scenarios/recover-final.json +10 -0
  146. package/test/scenarios/reroute-first.json +17 -0
  147. package/test/scenarios/reroute-second.json +16 -0
  148. package/test/scenarios/starvation-final.json +10 -0
  149. package/test/scenarios/starvation-promote.json +11 -0
  150. package/test/scenarios/wait-tools-continue.json +10 -0
  151. package/test/scenarios/wait-tools-xml.json +22 -0
  152. package/test/scenarios/wait-tools.json +15 -0
  153. package/test/session-format.test.ts +234 -0
  154. package/test/state.test.ts +335 -0
  155. package/test/stream-v02.test.ts +264 -0
  156. package/test/v031-display.test.ts +333 -0
  157. package/test/v032-conflict.test.ts +191 -0
  158. package/test/v032-truncation.test.ts +291 -0
  159. package/ui/client.js +576 -0
  160. package/ui/index.js +282 -0
  161. package/ui/package.json +26 -0
@@ -0,0 +1,200 @@
1
+ // eval/tools/machine-test.cjs —— 实机对照:默认模式下「关插件 / 开插件」各跑同一个项目一次
2
+ //
3
+ // 用法:
4
+ // node eval/tools/machine-test.cjs --plan # 只打印计划,不调模型、不读 key
5
+ // node eval/tools/machine-test.cjs --key <DEEPSEEK_API_KEY> # 真跑(两臂串行)
6
+ // node eval/tools/machine-test.cjs --key sk-xxx --task C1 --stamp 2026-11-17a --timeout 900
7
+ //
8
+ // 两臂(与 R3 评测矩阵同一套装置,控制变量不变):
9
+ // baseline = patches/eval-baseline.yml —— 停用本插件五个 entry + 恢复内建 agent-loop(插件关)
10
+ // stream = patches/eval-stream.yml —— 本插件全量接管,默认配置(balanced,enableCode: false)
11
+ // 两臂同装 eval-probe(只读透传),任务文本 / seed / 判题器逐字节相同。
12
+ //
13
+ // 产物:eval/runs/<stamp>/{baseline,stream}/<TASK>/{ws,probe,metrics.json,session-log}
14
+ // eval/runs/<stamp>/summary.json(M1 四分量)
15
+ // eval/runs/<stamp>/RESULTS.md(逐任务对照,--out 写入,不覆盖 eval/RESULTS.md)
16
+ // eval/runs/<stamp>/machine-test-report.md(本工具生成的单项目实机报告)
17
+ //
18
+ // 红线:key 只在子进程 env 里出现,不落盘、不打日志(只印后 4 位);不写 eval/RESULTS.md。
19
+
20
+ const fs = require('node:fs');
21
+ const path = require('node:path');
22
+ const { spawnSync } = require('node:child_process');
23
+
24
+ const PKG_ROOT = path.resolve(__dirname, '..', '..');
25
+ const TASKS_FILE = path.join(PKG_ROOT, 'eval', 'tasks', 'tasks.json');
26
+
27
+ function arg(name, def) {
28
+ const i = process.argv.indexOf(`--${name}`);
29
+ return i >= 0 && i + 1 < process.argv.length ? process.argv[i + 1] : def;
30
+ }
31
+
32
+ const flag = (name) => process.argv.includes(`--${name}`);
33
+ const today = () => new Date().toISOString().slice(0, 10);
34
+
35
+ // --runs:产物根目录(默认 eval/runs);--report-only:只从既有 metrics 重出报告,不跑模型。
36
+ const RUNS = arg('runs') ? path.resolve(arg('runs')) : path.join(PKG_ROOT, 'eval', 'runs');
37
+ const reportOnly = flag('report-only');
38
+
39
+ const taskId = arg('task', 'R2');
40
+ const stamp = arg('stamp', `${today()}-machine`);
41
+ const timeout = Number(arg('timeout', '900'));
42
+ const key = arg('key', '') || process.env.DEEPSEEK_API_KEY || '';
43
+ const force = flag('force');
44
+ const planOnly = flag('plan');
45
+
46
+ function die(msg, code = 2) {
47
+ console.error(`[machine-test] ${msg}`);
48
+ process.exit(code);
49
+ }
50
+
51
+ function tail4(s) {
52
+ return s ? `****${s.slice(-4)}` : '<unset>';
53
+ }
54
+
55
+ // ---- 任务与补丁 ----
56
+ const tasksDoc = JSON.parse(fs.readFileSync(TASKS_FILE, 'utf8'));
57
+ const task = tasksDoc.tasks.find((t) => t.id === taskId);
58
+ if (!task) die(`未知任务 ${taskId};可选:${tasksDoc.tasks.map((t) => t.id).join(', ')}`);
59
+
60
+ const PATCH = {
61
+ baseline: path.join(PKG_ROOT, 'patches', 'eval-baseline.yml'),
62
+ stream: path.join(PKG_ROOT, 'patches', 'eval-stream.yml'),
63
+ };
64
+
65
+ // ---- 计划输出 ----
66
+ const runDir = path.join(RUNS, stamp);
67
+ const steps = [
68
+ `preflight:node ${process.version} / dsh 版本 / dist 在位 / key ${planOnly ? '<未读取>' : tail4(key)}`,
69
+ `baseline:dsh --profile headless --patch patches/eval-baseline.yml headless "<${task.id} prompt>"(插件关)`,
70
+ `stream :dsh --profile headless --patch patches/eval-stream.yml headless "<${task.id} prompt>"(插件开,默认配置)`,
71
+ `summarize:node eval/tools/summarize.cjs --stamp ${stamp} --md`,
72
+ `report :node eval/tools/gen-results.cjs --stamp ${stamp} --out eval/runs/${stamp}/RESULTS.md`,
73
+ ];
74
+
75
+ console.log('[machine-test] 任务:', `${task.id}(${task.type} / ${task.title})`);
76
+ console.log('[machine-test] stamp:', stamp, '| 每臂超时:', timeout, 's | key:', planOnly ? '<plan 不读取>' : tail4(key));
77
+ console.log('[machine-test] 计划:');
78
+ steps.forEach((s, i) => console.log(` ${i + 1}. ${s}`));
79
+
80
+ if (planOnly) {
81
+ console.log(`[machine-test] --plan:只打印计划,未调用模型。真跑去掉 --plan 并给 --key(或设 DEEPSEEK_API_KEY)。`);
82
+ console.log(`[machine-test] 预计花费:2 次单任务运行(本机 v02 批次同题 baseline 约 3.8 万 token、stream 约 1.5 万,仅供量级参考)。`);
83
+ process.exit(0);
84
+ }
85
+
86
+ if (!reportOnly) {
87
+ // ---- preflight ----
88
+ if (!key) die('缺少 key:传 --key <KEY> 或设环境变量 DEEPSEEK_API_KEY');
89
+ for (const [name, p] of Object.entries(PATCH)) if (!fs.existsSync(p)) die(`补丁缺失:${name} -> ${p}`);
90
+ if (!fs.existsSync(path.join(PKG_ROOT, 'dist', 'entry', 'loop.js'))) die('dist 未构建:先 npm run build');
91
+
92
+ const dshVersion = spawnSync('dsh', ['--version'], { shell: true, encoding: 'utf8', windowsHide: true });
93
+ if (dshVersion.status !== 0) die(`dsh 不可用:${(dshVersion.stderr || dshVersion.error || '').toString().slice(0, 200)}`);
94
+ console.log('[machine-test] dsh', (dshVersion.stdout || '').trim());
95
+
96
+ if (fs.existsSync(runDir) && !force) {
97
+ die(`eval/runs/${stamp} 已存在;换 --stamp,或加 --force 覆盖(会删掉该目录下同名两臂产物)`);
98
+ }
99
+ }
100
+
101
+ // ---- 跑两臂 ----
102
+ function runArm(arm) {
103
+ console.log(`\n[machine-test] ===== ${arm} 开始 =====`);
104
+ const started = Date.now();
105
+ const r = spawnSync(
106
+ process.execPath,
107
+ [path.join(__dirname, 'runner.cjs'), '--arm', arm, '--stamp', stamp, '--tasks', taskId, '--key', key, '--timeout', String(timeout)],
108
+ { cwd: PKG_ROOT, stdio: 'inherit', windowsHide: true },
109
+ );
110
+ const seconds = Math.round((Date.now() - started) / 1000);
111
+ console.log(`[machine-test] ===== ${arm} 结束:exit=${r.status} 用时 ${seconds}s =====`);
112
+ return { status: r.status, seconds };
113
+ }
114
+
115
+ const runs = reportOnly
116
+ ? { baseline: { seconds: '—' }, stream: { seconds: '—' } }
117
+ : { baseline: runArm('baseline'), stream: runArm('stream') };
118
+
119
+ // ---- 汇总(M1 四分量)----
120
+ if (!reportOnly) {
121
+ const summarize = spawnSync(process.execPath, [path.join(__dirname, 'summarize.cjs'), '--stamp', stamp, '--md'], {
122
+ cwd: PKG_ROOT, encoding: 'utf8', windowsHide: true,
123
+ });
124
+ if (summarize.status !== 0) console.error('[machine-test] summarize 失败:', (summarize.stderr || '').slice(-500));
125
+ else process.stdout.write(summarize.stdout);
126
+
127
+ // ---- 逐任务对照(写到 runs 目录,不碰 eval/RESULTS.md)----
128
+ const genOut = path.join(runDir, 'RESULTS.md');
129
+ const gen = spawnSync(process.execPath, [path.join(__dirname, 'gen-results.cjs'), '--stamp', stamp, '--out', genOut], {
130
+ cwd: PKG_ROOT, encoding: 'utf8', windowsHide: true,
131
+ });
132
+ if (gen.status !== 0) console.error('[machine-test] gen-results 失败:', (gen.stderr || '').slice(-500));
133
+ }
134
+
135
+ // ---- 单项目报告 ----
136
+ function readMetrics(arm) {
137
+ const p = path.join(runDir, arm, taskId, 'metrics.json');
138
+ if (!fs.existsSync(p)) return null;
139
+ return JSON.parse(fs.readFileSync(p, 'utf8'));
140
+ }
141
+
142
+ const m = { baseline: readMetrics('baseline'), stream: readMetrics('stream') };
143
+ const total = (x) => (x ? x.M1_tokens.input + x.M1_tokens.cacheRead + x.M1_tokens.cacheWrite + x.M1_tokens.output : null);
144
+ const tb = total(m.baseline);
145
+ const ts = total(m.stream);
146
+ const ratio = tb && ts ? ((ts / tb) * 100).toFixed(1) : 'n/a';
147
+
148
+ const rows = [
149
+ ['臂', 'exit', '请求数', '输入(未缓存)', '缓存读', '缓存写', '输出', '总 token', 'checks'],
150
+ ['baseline(插件关)', m.baseline?.exitCode ?? '未跑', m.baseline?.M2_callCount.probe ?? '—',
151
+ m.baseline?.M1_tokens.input ?? '—', m.baseline?.M1_tokens.cacheRead ?? '—', m.baseline?.M1_tokens.cacheWrite ?? '—',
152
+ m.baseline?.M1_tokens.output ?? '—', tb ?? '—', m.baseline ? (m.baseline.checksPass ? 'PASS' : 'FAIL') : '—'],
153
+ ['stream(插件开)', m.stream?.exitCode ?? '未跑', m.stream?.M2_callCount.probe ?? '—',
154
+ m.stream?.M1_tokens.input ?? '—', m.stream?.M1_tokens.cacheRead ?? '—', m.stream?.M1_tokens.cacheWrite ?? '—',
155
+ m.stream?.M1_tokens.output ?? '—', ts ?? '—', m.stream ? (m.stream.checksPass ? 'PASS' : 'FAIL') : '—'],
156
+ ];
157
+
158
+ console.log('\n[machine-test] 单项目实机结果(四分量口径,与 eval/METRICS.md M1 一致)');
159
+ for (const r of rows) console.log(' ' + r.map((c) => String(c)).join(' | '));
160
+ console.log(`[machine-test] 对照/基线 总 token 比:${ratio}%`);
161
+
162
+ const report = [
163
+ `# 实机对照报告(单项目 / 默认模式):${task.id}`,
164
+ '',
165
+ `- 生成:\`eval/tools/machine-test.cjs\`,stamp \`${stamp}\`,${new Date().toISOString()}`,
166
+ `- 任务:${task.id}(${task.type} / ${task.title}),题面见 \`eval/tasks/tasks.json\``,
167
+ '- 两臂:baseline = 插件关(内建 agent-loop),stream = 插件开(默认配置,balanced,`enableCode: false`)',
168
+ '- 口径:总 token = Σ(input + cacheRead + cacheWrite + output),逐请求取 probe usage 原文;标题旁路请求两臂同剔',
169
+ '',
170
+ '| 臂 | exit | 请求数 | 输入(未缓存) | 缓存读 | 缓存写 | 输出 | 总 token | checks | 用时 |',
171
+ '|---|---:|---:|---:|---:|---:|---:|---:|---|---:|',
172
+ ...['baseline', 'stream'].map((arm) => {
173
+ const x = m[arm];
174
+ return x
175
+ ? `| ${arm} | ${x.exitCode} | ${x.M2_callCount.probe} | ${x.M1_tokens.input} | ${x.M1_tokens.cacheRead} | ${x.M1_tokens.cacheWrite} | ${x.M1_tokens.output} | ${total(x)} | ${x.checksPass ? 'PASS' : 'FAIL'} | ${runs[arm].seconds}s |`
176
+ : `| ${arm} | 未跑 | — | — | — | — | — | — | — | ${runs[arm].seconds}s |`;
177
+ }),
178
+ '',
179
+ `**对照/基线总 token 比:${ratio}%**`,
180
+ '',
181
+ '## 如实说明',
182
+ '',
183
+ '- 单样本只作方向参考:一次运行受模型采样、缓存命中与网络波动影响,**不能替代 R3 十题矩阵**,也不能据此改写 README 的 28.9%。',
184
+ '- `exit=1` 若同时 `checks=PASS`,多半是重入预算护栏耗尽(见 docs/limitations.md 第 2 条),不是任务失败;报告里如实保留原始 exit。',
185
+ '- 两臂 model/provider 若不一致,本报告无效(gen-results 会在逐任务表标注「作废重跑」)。',
186
+ '- 证据链:`eval/runs/' + stamp + '/{baseline,stream}/' + task.id + '/{probe,metrics.json,session-log}`。',
187
+ '',
188
+ `逐任务对照(含 injected_ratio / prefix 取证):[RESULTS.md](./RESULTS.md)`,
189
+ ].join('\n');
190
+
191
+ const reportPath = path.join(runDir, 'machine-test-report.md');
192
+ fs.mkdirSync(runDir, { recursive: true });
193
+ fs.writeFileSync(reportPath, report, 'utf8');
194
+ console.log(`[machine-test] 报告:${path.relative(PKG_ROOT, reportPath) || reportPath}`);
195
+
196
+ const missing = !m.baseline || !m.stream;
197
+ if (missing) {
198
+ console.error('[machine-test] 有一臂没有产出 metrics.json,报告不完整;请看上面各臂的 stderr。');
199
+ process.exit(1);
200
+ }
@@ -0,0 +1,59 @@
1
+ # tf-code 真机五场景(K6,人工执行;每场景记录输入 / kind / 账本 / 最终状态)
2
+
3
+ > 前提:宿主挂载 `@deepseek-ai/dsh-ptc-runtime`,装配 `enableCode: true`(或 `power` 模式)。
4
+ > 每个场景跑完记录:任务文本、结果 kind、`plugin:tf-agent/command` 账本行(sandbox 事实)、
5
+ > 最终状态(completed / blocked / error)。mock 不替代真机(K6 红线)。
6
+ > 预算参考:每场景约 0.5–2 元(deepseek-flash;见 docs/TEST-PROPOSITIONS.md 口径)。
7
+
8
+ ## S1 成功(happy path)
9
+
10
+ ```
11
+ 请用 tf-code 统计 data/sales-q1.csv 的行数与"amount"列的总和,把结果写进 out/summary.md(两行:total=、sum=)。
12
+ ```
13
+
14
+ - 期望:`tf-code` ok;`enforcement` 记账在账本;summary.md 内容正确;completed。
15
+ - 取证:账本 `mode`/`enforcement`;driver-*.json 里 command c1 ok。
16
+
17
+ ## S2 失败(代码错误 → 结构化 kind)
18
+
19
+ ```
20
+ 请用 tf-code 读取 data/sales-q1.csv 并统计,把结果写进 out/summary.md。(程序里故意写错:调用不存在的 tools.parseCsv)
21
+ ```
22
+
23
+ - 期望:失败注入 `[tf-stream:code-error] kind=exception retry=yes`(或 invalid-output);
24
+ 模型修复后重发(换新 id)或改道 tf-bash;不允许原样重跑拿到相同错误后无进展循环。
25
+
26
+ ## S3 取消(K4)
27
+
28
+ ```
29
+ (发起一个 tf-code 长程序后,在运行中按 Esc / 发取消)
30
+ ```
31
+
32
+ - 期望:`kind=abort`,abort ≤500ms 内传导;回合以 aborted 收束;**0 次后置写**
33
+ (取消后工作区不再出现新文件——对照工作区 manifest 前后快照)。
34
+ - 取证:账本 command failed kind=abort;turn-end reason aborted。
35
+
36
+ ## S4 沙箱拒绝(K2:拒绝与代码错误分开)
37
+
38
+ ```
39
+ (把 workspaceRoot 指到只读目录,或用 sandboxPolicy 拒绝写路径后)
40
+ 请用 tf-code 把结果写入 out/blocked.md。
41
+ ```
42
+
43
+ - 期望:绑定 write 失败 → 程序失败 `kind=exception`(宿主拒绝文本)或程序改道;
44
+ 账本记录 `denied` 与 `mode`;注入文案**不得**引导"修改代码重试"来解决权限问题
45
+ (拒绝 ≠ 代码错误,K3 红线)。
46
+
47
+ ## S5 partial(Windows enforcement 可见)
48
+
49
+ ```
50
+ (Windows 宿主 + ACL 档沙箱)请用 tf-code 在工作区内写 out/partial.md 并读回验证。
51
+ ```
52
+
53
+ - 期望:账本 `enforcement=partial` 显式可见;文档/注入不出现"已隔离/绝对只读"表述;
54
+ 结果按"文件效果部分强制"陈述。
55
+
56
+ ## 通过标准
57
+
58
+ 5/5 场景全部符合期望;声明面抽查:README / docs/limitations.md / docs/protocol.md
59
+ 三处无"隔离/等权"表述、`partial` 与 `danger-full-access` 均可见(K6 声明一致)。
@@ -0,0 +1,311 @@
1
+ // eval/tools/replay-check.cjs —— M5 回放一致性:按取证重放写入操作,比对工作区树哈希
2
+ //
3
+ // 用法:node eval/tools/replay-check.cjs --stamp <2026-11-14> [--tasks W1,R1]
4
+ //
5
+ // 重放来源(两臂各自的取证面):
6
+ // - 对照臂(stream):probe.jsonl 逐请求 messages 原文里的 tf-write 命令(协议文本,可靠)
7
+ // - 基线臂(baseline):session log assistant/attempt 流里的原生 tool_call(write 族)
8
+ // ——骨架来源是 v4 日志的 assistant/attempt chunk 流(同 06 驱动器累积逻辑)
9
+ // 重放目标:空 repl/ 目录先铺同一份 seed,再按记录顺序重放全部写入,然后
10
+ // repl/manifest.json 的树哈希 == 实跑后 ws/manifest.json 的树哈希 → M5 通过。
11
+ // 不可重放面(bash 命令、非 write 族工具)如实记 skipped,不假装通过。
12
+ //
13
+ // 用法(回填 metrics.json 的 replay 字段):
14
+ // node eval/tools/replay-check.cjs --stamp <stamp>
15
+
16
+ const fs = require('node:fs');
17
+ const path = require('node:path');
18
+ const { workspaceManifest, parseSessionLog } = require('./session-log.cjs');
19
+
20
+ const PKG_ROOT = path.resolve(__dirname, '..', '..');
21
+ const TASKS_FILE = path.join(PKG_ROOT, 'eval', 'tasks', 'tasks.json');
22
+ const FIXTURES = path.join(PKG_ROOT, 'eval', 'tasks', 'fixtures');
23
+
24
+ function arg(name, def) {
25
+ const i = process.argv.indexOf(`--${name}`);
26
+ return i >= 0 && i + 1 < process.argv.length ? process.argv[i + 1] : def;
27
+ }
28
+
29
+ /** 从 probe chunk 记录提取模型输出的写入类 tf-* 命令(对照臂)。
30
+ * 注意:必须取模型输出(chunk 流的 text 增量),不能取 request messages——
31
+ * 系统提示的协议示例文本里也有 <tf-write> 字样(smoke 实测误提取教训)。
32
+ *
33
+ * M3 修复(0.3.0):
34
+ * · 属性序无关——旧正则要求 path 在 id 之前,模型输出 id/depends_on 在前时提取 0 命中
35
+ * (replay 0/10 的直接根因);改为引号感知的头部扫描 + 通用属性解析。
36
+ * · 支持 tf-edit(replace)与 tf-append(追加/锚点插入)的回放,不再只认 tf-write。
37
+ * · 未闭合命令如实计入 errors(提取器出错 = "提取失败",不假装"回放通过")。 */
38
+ function extractTfWritesFromChunks(probeLines) {
39
+ const ops = [];
40
+ const errors = [];
41
+ const textBySeq = new Map(); // seq -> 聚合的模型文本
42
+ for (const l of probeLines) {
43
+ if (l.kind !== 'chunk') continue;
44
+ try {
45
+ const c = JSON.parse(l.chunk);
46
+ // dsh-llm 流词汇:text 增量字段是 `text`(text-delta / reasoning-delta 同形);
47
+ // 旧代码读 `textDelta` 永远命不中——replay 0/10 的第二个根因(2026-11-15 实测数据核对)。
48
+ if ((c.type === 'text' || c.type === 'text-delta') && typeof c.text === 'string') {
49
+ textBySeq.set(l.seq, (textBySeq.get(l.seq) || '') + c.text);
50
+ }
51
+ } catch { /* 原文保留,解析失败如实跳过 */ }
52
+ }
53
+ const seqs = [...textBySeq.keys()].sort((a, b) => a - b);
54
+ for (const seq of seqs) {
55
+ const text = textBySeq.get(seq);
56
+ const re = /<tf-(write|edit|append)(\s|\n)/g;
57
+ let m;
58
+ while ((m = re.exec(text)) !== null) {
59
+ const tag = `tf-${m[1]}`;
60
+ // 引号感知地找头部结束 '>'(属性值可含 >)
61
+ let i = m.index + tag.length + 1;
62
+ let inQuote = false;
63
+ let headerEnd = -1;
64
+ for (; i < text.length; i++) {
65
+ const ch = text[i];
66
+ if (inQuote) {
67
+ if (ch === '"') inQuote = false;
68
+ continue;
69
+ }
70
+ if (ch === '"') inQuote = true;
71
+ else if (ch === '>') { headerEnd = i; break; }
72
+ }
73
+ if (headerEnd === -1) {
74
+ errors.push({ seq, tag, reason: 'unclosed-header' });
75
+ continue;
76
+ }
77
+ const header = text.slice(m.index, headerEnd + 1);
78
+ if (header.endsWith('/>')) {
79
+ errors.push({ seq, tag, reason: 'self-closing-write-command' });
80
+ continue;
81
+ }
82
+ // 通用属性解析(键序无关)
83
+ const attrs = {};
84
+ const attrRe = /([a-zA-Z][a-zA-Z0-9_-]*)\s*=\s*"([^"]*)"/g;
85
+ let am;
86
+ while ((am = attrRe.exec(header)) !== null) attrs[am[1]] = am[2];
87
+ const bodyStart = headerEnd + 1;
88
+ const closeTag = `</${tag}>`;
89
+ const closeIdx = text.indexOf(closeTag, bodyStart);
90
+ if (closeIdx === -1) {
91
+ errors.push({ seq, tag, reason: 'unclosed-body' });
92
+ continue;
93
+ }
94
+ const body = text.slice(bodyStart, closeIdx);
95
+ re.lastIndex = closeIdx + closeTag.length;
96
+ if (attrs['path'] === undefined) { errors.push({ seq, tag, reason: 'missing-path' }); continue; }
97
+ // 绝对路径处理:模型把任务工作区绝对路径回显时,取 '/ws/' 之后的部分相对化;
98
+ // 无法相对化(工作区外路径)→ 提取失败如实记录,不静默丢。
99
+ let relPath = attrs['path'];
100
+ const wsSplit = relPath.replace(/\\/g, '/').split(/\/ws\//);
101
+ if (/^([a-zA-Z]:[\\/]|\/)/.test(relPath)) {
102
+ if (wsSplit.length > 1) relPath = wsSplit[wsSplit.length - 1];
103
+ else { errors.push({ seq, tag, reason: `absolute-path-outside-workspace:${attrs['path'].slice(0, 60)}` }); continue; }
104
+ }
105
+ if (tag === 'tf-write') {
106
+ ops.push({ path: relPath, content: body, source: `tf-write@seq${seq}` });
107
+ } else if (tag === 'tf-edit') {
108
+ const split = splitEditBody(body);
109
+ if (split === null) { errors.push({ seq, tag, reason: 'bad-edit-markers' }); continue; }
110
+ ops.push({ path: relPath, replace: { old: split.old, new: split.new }, source: `tf-edit@seq${seq}` });
111
+ } else {
112
+ ops.push({ path: relPath, append: body, source: `tf-append@seq${seq}` });
113
+ }
114
+ }
115
+ }
116
+ if (errors.length > 0) ops.push({ __errors: errors });
117
+ return ops;
118
+ }
119
+
120
+ /** tf-edit body 的 OLD/NEW 标记切分(与 src/kernel/parser.ts 的 splitEditBody 同语义)。
121
+ * 本文件是 CJS 工具,不 import TS 内核——两处语义必须同步,改一处即改另一处。 */
122
+ function splitEditBody(body) {
123
+ const lines = body.split('\n');
124
+ const idxOld = lines.indexOf('<<<<<<< OLD');
125
+ const idxSplit = lines.indexOf('=======');
126
+ const idxEnd = lines.indexOf('>>>>>>> NEW');
127
+ if (idxOld === -1 || idxSplit === -1 || idxEnd === -1) return null;
128
+ if (!(idxOld < idxSplit && idxSplit < idxEnd)) return null;
129
+ if (lines.indexOf('<<<<<<< OLD', idxOld + 1) !== -1) return null;
130
+ if (lines.indexOf('=======', idxSplit + 1) !== -1) return null;
131
+ if (lines.indexOf('>>>>>>> NEW', idxEnd + 1) !== -1) return null;
132
+ return { old: lines.slice(idxOld + 1, idxSplit).join('\n'), new: lines.slice(idxSplit + 1, idxEnd).join('\n') };
133
+ }
134
+
135
+ /** 从 session log assistant/attempt 流累积原生 tool_call(基线臂)。 */
136
+ function extractNativeWrites(records) {
137
+ const calls = new Map(); // id -> {name, arguments}
138
+ for (const r of records) {
139
+ if (r.type !== 'assistant/attempt') continue;
140
+ for (const s of (r.data && r.data.stream) || []) {
141
+ const c = s.chunk || {};
142
+ if (c.type === 'tool-call-delta' && c.id !== undefined) {
143
+ const acc = calls.get(c.id) || { name: undefined, args: '' };
144
+ if (c.name !== undefined) acc.name = c.name;
145
+ if (typeof c.argumentsDelta === 'string') acc.args += c.argumentsDelta;
146
+ calls.set(c.id, acc);
147
+ } else if (c.type === 'tool-call' && c.block && c.block.id !== undefined) {
148
+ const acc = calls.get(c.block.id) || { name: undefined, args: '' };
149
+ if (c.block.name !== undefined) acc.name = c.block.name;
150
+ if (typeof c.block.arguments === 'string' && acc.args.length === 0) acc.args = c.block.arguments;
151
+ calls.set(c.block.id, acc);
152
+ }
153
+ }
154
+ }
155
+ const ops = [];
156
+ const skipped = [];
157
+ for (const [id, acc] of calls) {
158
+ let args = {};
159
+ try { args = JSON.parse(acc.args || '{}'); } catch { args = null; }
160
+ if (acc.name === 'write' && args && typeof args.file_path === 'string') {
161
+ ops.push({ path: args.file_path, content: String(args.content ?? ''), source: `native:${acc.name}` });
162
+ } else if (acc.name === 'str_replace_editor' && args && typeof args.path === 'string' && args.command === 'create') {
163
+ ops.push({ path: args.path, content: String(args.file_text ?? ''), source: `native:${acc.name}` });
164
+ } else if (acc.name === 'str_replace_editor' && args && typeof args.path === 'string') {
165
+ // str_replace:读取当前内容做替换(replay 目录中 seed 已铺,顺序回放即可)
166
+ const rel = normRel(args.path);
167
+ const cur = existing.get(rel);
168
+ if (cur !== undefined && typeof args.old_string === 'string') {
169
+ ops.push({ path: rel, replace: { old: args.old_string, new: String(args.new_string ?? '') }, source: `native:${acc.name}` });
170
+ } else skipped.push({ id, name: acc.name, reason: 'replace-target-missing' });
171
+ } else {
172
+ skipped.push({ id, name: acc.name, reason: acc.name === undefined ? 'incomplete-call' : 'not-a-write-tool' });
173
+ }
174
+ }
175
+ return { ops, skipped };
176
+ }
177
+
178
+ const existing = new Map(); // replay 目录当前内容(rel -> string),供 str_replace 顺序回放
179
+
180
+ function normRel(p) {
181
+ return String(p).replace(/\\/g, '/').replace(/^\.\//, '').replace(/^\//, '');
182
+ }
183
+
184
+ function seedWorkspace(replDir, task) {
185
+ fs.rmSync(replDir, { recursive: true, force: true });
186
+ fs.mkdirSync(replDir, { recursive: true });
187
+ for (const s of task.seed || []) {
188
+ const dst = path.join(replDir, s.path);
189
+ fs.mkdirSync(path.dirname(dst), { recursive: true });
190
+ fs.copyFileSync(path.join(FIXTURES, s.fixture), dst);
191
+ }
192
+ existing.clear();
193
+ (function walk(dir) {
194
+ for (const name of fs.readdirSync(dir)) {
195
+ const p = path.join(dir, name);
196
+ if (fs.statSync(p).isDirectory()) walk(p);
197
+ else existing.set(path.relative(replDir, p).replace(/\\/g, '/'), fs.readFileSync(p, 'utf8'));
198
+ }
199
+ })(replDir);
200
+ }
201
+
202
+ function applyOp(op) {
203
+ const rel = normRel(op.path);
204
+ if (op.replace) {
205
+ const cur = existing.get(rel);
206
+ if (cur === undefined) return `replace 目标不存在: ${rel}`;
207
+ const next = cur.split(op.replace.old).join(op.replace.new);
208
+ if (next === cur) return `replace 未命中(old 不在文件中): ${rel}`;
209
+ existing.set(rel, next);
210
+ return null;
211
+ }
212
+ if (op.append) {
213
+ // tf-append 语义:锚点插入(op.anchor 给定时替换 anchor → anchor+正文)或 EOF 追加。
214
+ if (op.anchor) {
215
+ const cur = existing.get(rel);
216
+ if (cur === undefined) return `append 锚点目标不存在: ${rel}`;
217
+ if (!cur.includes(op.anchor)) return `append 锚点未命中: ${rel}`;
218
+ existing.set(rel, cur.replace(op.anchor, `${op.anchor}\n${op.append}`));
219
+ return null;
220
+ }
221
+ const cur = existing.get(rel);
222
+ existing.set(rel, cur === undefined ? op.append : `${cur}\n${op.append}`);
223
+ return null;
224
+ }
225
+ existing.set(rel, String(op.content));
226
+ return null;
227
+ }
228
+
229
+ function flush(replDir) {
230
+ for (const [rel, content] of existing) {
231
+ const p = path.join(replDir, rel);
232
+ fs.mkdirSync(path.dirname(p), { recursive: true });
233
+ fs.writeFileSync(p, content, 'utf8');
234
+ }
235
+ }
236
+
237
+ function checkTask(task, runDir, arm, taskId) {
238
+ const taskDir = path.join(runDir, arm, taskId);
239
+ const metricsPath = path.join(taskDir, 'metrics.json');
240
+ if (!fs.existsSync(metricsPath)) return { taskId, error: 'metrics.json 不存在(未跑)' };
241
+ const metrics = JSON.parse(fs.readFileSync(metricsPath, 'utf8'));
242
+
243
+ const replDir = path.join(taskDir, 'repl');
244
+ seedWorkspace(replDir, task);
245
+ let ops = [];
246
+ let extractErrors = [];
247
+ const skipped = [];
248
+ if (arm === 'stream') {
249
+ const probeFile = path.join(taskDir, 'probe', 'probe.jsonl');
250
+ if (!fs.existsSync(probeFile)) return { taskId, error: 'probe.jsonl 不存在' };
251
+ const probeLines = fs.readFileSync(probeFile, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return { kind: 'PARSE_ERROR' }; } });
252
+ ops = extractTfWritesFromChunks(probeLines);
253
+ // 提取器自身的错误(未闭合/缺属性等):报告"提取失败",不得记成"回放通过"
254
+ extractErrors = ops.filter((o) => o.__errors !== undefined).flatMap((o) => o.__errors);
255
+ ops = ops.filter((o) => o.__errors === undefined);
256
+ } else {
257
+ const logPath = path.join(taskDir, 'session-log', 'session.v4.jsonl.zstd');
258
+ if (!fs.existsSync(logPath)) return { taskId, error: 'session log 存档不存在' };
259
+ const records = parseSessionLog(logPath);
260
+ const extracted = extractNativeWrites(records);
261
+ ops = extracted.ops;
262
+ skipped.push(...extracted.skipped);
263
+ }
264
+
265
+ const errors = [...extractErrors];
266
+ for (const op of ops) {
267
+ const e = applyOp(op);
268
+ if (e) errors.push(e);
269
+ }
270
+ flush(replDir);
271
+ const manifest = workspaceManifest(replDir);
272
+ fs.writeFileSync(path.join(replDir, 'manifest.json'), JSON.stringify(manifest, null, 2), 'utf8');
273
+ const wsManifest = JSON.parse(fs.readFileSync(path.join(taskDir, 'ws', 'manifest.json'), 'utf8'));
274
+ const hashMatch = manifest.root === wsManifest.root;
275
+ // M3 语义(alpha.1):零写入任务在种子树 == 结果树时通过(不得因 0 操作判失败);
276
+ // 提取器出错 = "提取失败"(不通过);哈希不一致 = 存在未回放的写入差异(不通过)。
277
+ const pass = extractErrors.length === 0 && errors.length === 0 && hashMatch;
278
+ const replay = {
279
+ opsReplayed: ops.length, skipped, errors,
280
+ replRoot: manifest.root,
281
+ wsRoot: wsManifest.root,
282
+ pass,
283
+ note: extractErrors.length > 0
284
+ ? `提取失败 ${extractErrors.length} 条(未闭合/缺属性等):不记为回放通过`
285
+ : errors.length > 0
286
+ ? `回放应用出错 ${errors.length} 条`
287
+ : ops.length === 0
288
+ ? (hashMatch ? '零写入任务:种子树 == 结果树(通过,非"提取失败")' : '零写入但树哈希不一致:存在未回放的写入差异')
289
+ : (hashMatch ? '重放树哈希 == 实跑树哈希' : '哈希不一致,见 repl/ 与 ws/ manifest'),
290
+ };
291
+ metrics.replay = replay;
292
+ fs.writeFileSync(metricsPath, JSON.stringify(metrics, null, 2), 'utf8');
293
+ console.log(`[${arm}/${taskId}] replay ops=${replay.opsReplayed} skipped=${skipped.length} pass=${replay.pass}`);
294
+ return { taskId, replay };
295
+ }
296
+
297
+ function main() {
298
+ const stamp = arg('stamp');
299
+ if (!stamp) { console.error('--stamp 必填'); process.exit(2); }
300
+ const runDir = path.join(PKG_ROOT, 'eval', 'runs', stamp);
301
+ const all = JSON.parse(fs.readFileSync(TASKS_FILE, 'utf8'));
302
+ const pick = arg('tasks', 'all') === 'all' ? all.tasks : all.tasks.filter((t) => arg('tasks', '').split(',').includes(t.id));
303
+ for (const arm of ['baseline', 'stream']) {
304
+ for (const t of pick) {
305
+ if (fs.existsSync(path.join(runDir, arm, t.id, 'metrics.json'))) checkTask(t, runDir, arm, t.id);
306
+ }
307
+ }
308
+ }
309
+
310
+ if (require.main === module) main();
311
+ module.exports = { extractTfWritesFromChunks, extractNativeWrites, checkTask };