@hugesb/dsh-stream-agent 0.0.0-stage → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +622 -0
- package/CONTRIBUTING.md +72 -0
- package/LICENSE +202 -0
- package/README.en.md +349 -0
- package/README.md +300 -3
- package/cordis.patch.yml +77 -0
- package/dist/entry/action-gate.js +206 -0
- package/dist/entry/adapter-kit.js +192 -0
- package/dist/entry/context-render.js +119 -0
- package/dist/entry/control-bar-cli.js +289 -0
- package/dist/entry/control-bar.js +123 -0
- package/dist/entry/control-session.js +115 -0
- package/dist/entry/control-state.js +125 -0
- package/dist/entry/cordis-compat.js +95 -0
- package/dist/entry/eval-probe.js +112 -0
- package/dist/entry/intent-log.js +121 -0
- package/dist/entry/json-persistence.js +306 -0
- package/dist/entry/loop.js +4163 -0
- package/dist/entry/mock-provider.js +549 -0
- package/dist/entry/obs-broker.js +178 -0
- package/dist/entry/ptc-bridge.js +405 -0
- package/dist/entry/state-handle.js +217 -0
- package/dist/entry/takeover-registry.js +60 -0
- package/dist/kernel/control-bar.js +559 -0
- package/dist/kernel/final.js +62 -0
- package/dist/kernel/gate.js +464 -0
- package/dist/kernel/modes.js +253 -0
- package/dist/kernel/obs.js +517 -0
- package/dist/kernel/parser.js +1081 -0
- package/dist/kernel/render.js +466 -0
- package/dist/kernel/state.js +416 -0
- package/dist/version.js +9 -0
- package/docs/TEST-PROPOSITIONS.md +139 -0
- package/docs/adr-001-segment-boundary.md +61 -0
- package/docs/architecture.md +104 -0
- package/docs/desktop-ui-feasibility.md +277 -0
- package/docs/disabled-lines-audit.md +53 -0
- package/docs/limitations.md +181 -0
- package/docs/protocol.md +99 -0
- package/docs/v0.4.0-plan.md +50 -0
- package/eval/METRICS.md +128 -0
- package/eval/RESULTS.md +246 -0
- package/eval/tasks/fixtures/big-report.md +298 -0
- package/eval/tasks/fixtures/build-manifest.txt +5 -0
- package/eval/tasks/fixtures/deploy.yaml +4 -0
- package/eval/tasks/fixtures/error.log +200 -0
- package/eval/tasks/fixtures/features.yaml +4 -0
- package/eval/tasks/fixtures/pipeline.yaml +4 -0
- package/eval/tasks/fixtures/sales-q1.csv +61 -0
- package/eval/tasks/fixtures/settings.yaml +4 -0
- package/eval/tasks/fixtures/users-roster.json +304 -0
- package/eval/tasks/fixtures/users-scores.json +20 -0
- package/eval/tasks/gen-fixtures.cjs +80 -0
- package/eval/tasks/tasks.json +150 -0
- package/eval/tools/f6-kill-drill.cjs +176 -0
- package/eval/tools/f6-kill-drill.worker.cjs +131 -0
- package/eval/tools/gen-results.cjs +191 -0
- package/eval/tools/machine-test.cjs +200 -0
- package/eval/tools/manual-code-scenarios.md +59 -0
- package/eval/tools/replay-check.cjs +311 -0
- package/eval/tools/runner.cjs +219 -0
- package/eval/tools/session-log.cjs +106 -0
- package/eval/tools/summarize.cjs +185 -0
- package/eval/tools/t3-cells.cjs +173 -0
- package/package.json +73 -6
- package/patches/adapters-smoke.yml +41 -0
- package/patches/eval-account.yml +36 -0
- package/patches/eval-baseline.yml +33 -0
- package/patches/eval-stream-code-off.yml +46 -0
- package/patches/eval-stream-code-on.yml +47 -0
- package/patches/eval-stream.yml +54 -0
- package/patches/takeover.yml +42 -0
- package/patches/test-preset.yml +23 -0
- package/preset/stream/cordis.patch.yml +39 -0
- package/preset/stream/package.json +13 -0
- package/preset.yml +56 -0
- package/profiles/stream/README.md +66 -0
- package/profiles/stream/cordis.patch.yml +12 -0
- package/profiles/stream/package.json +17 -0
- package/profiles/stream-headless/package.json +16 -0
- package/scripts/build.mjs +103 -0
- package/scripts/check-consistency.mjs +146 -0
- package/scripts/gate.mjs +138 -0
- package/scripts/install-check.mjs +116 -0
- package/scripts/install-cycle.mjs +88 -0
- package/scripts/release.mjs +428 -0
- package/scripts/repair-session-logs.mjs +498 -0
- package/src/entry/action-gate.ts +206 -0
- package/src/entry/adapter-kit.ts +192 -0
- package/src/entry/context-render.ts +119 -0
- package/src/entry/control-bar-cli.ts +289 -0
- package/src/entry/control-bar.ts +123 -0
- package/src/entry/control-session.ts +115 -0
- package/src/entry/control-state.ts +125 -0
- package/src/entry/cordis-compat.ts +95 -0
- package/src/entry/eval-probe.ts +112 -0
- package/src/entry/intent-log.ts +121 -0
- package/src/entry/json-persistence.ts +306 -0
- package/src/entry/loop.ts +4163 -0
- package/src/entry/mock-provider.ts +549 -0
- package/src/entry/obs-broker.ts +178 -0
- package/src/entry/ptc-bridge.ts +405 -0
- package/src/entry/state-handle.ts +217 -0
- package/src/entry/takeover-registry.ts +60 -0
- package/src/kernel/control-bar.ts +559 -0
- package/src/kernel/final.ts +62 -0
- package/src/kernel/gate.ts +464 -0
- package/src/kernel/modes.ts +253 -0
- package/src/kernel/obs.ts +517 -0
- package/src/kernel/parser.ts +1081 -0
- package/src/kernel/render.ts +466 -0
- package/src/kernel/state.ts +416 -0
- package/src/version.ts +9 -0
- package/test/activation.test.ts +187 -0
- package/test/adapters.test.ts +676 -0
- package/test/control-bar.test.ts +486 -0
- package/test/desktop-ui.test.ts +359 -0
- package/test/eval-probe.test.ts +139 -0
- package/test/f5-plan.test.ts +221 -0
- package/test/f6-intent.test.ts +168 -0
- package/test/f7-segment.test.ts +190 -0
- package/test/fix-limits.test.ts +599 -0
- package/test/gate.test.ts +380 -0
- package/test/helpers/stub-loop.ts +240 -0
- package/test/host-conformance.test.ts +418 -0
- package/test/kernel/obs.test.ts +329 -0
- package/test/kernel/parser.test.ts +451 -0
- package/test/loop.test.ts +1393 -0
- package/test/mock-provider.test.ts +386 -0
- package/test/ptc-bridge.test.ts +356 -0
- package/test/render.test.ts +418 -0
- package/test/s2-install-check.test.ts +46 -0
- package/test/scenarios/3-file-basic.json +13 -0
- package/test/scenarios/5-file-basic.json +22 -0
- package/test/scenarios/burn-more.json +9 -0
- package/test/scenarios/degraded-final.json +10 -0
- package/test/scenarios/fault-abort-xml.json +11 -0
- package/test/scenarios/fault-malformed-command.json +12 -0
- package/test/scenarios/fault-midstream-abort.json +11 -0
- package/test/scenarios/fault-saturating-output.json +9 -0
- package/test/scenarios/fault-timeout.json +9 -0
- package/test/scenarios/peek-first.json +11 -0
- package/test/scenarios/peek-second.json +11 -0
- package/test/scenarios/peek-third.json +10 -0
- package/test/scenarios/recover-final.json +10 -0
- package/test/scenarios/reroute-first.json +17 -0
- package/test/scenarios/reroute-second.json +16 -0
- package/test/scenarios/starvation-final.json +10 -0
- package/test/scenarios/starvation-promote.json +11 -0
- package/test/scenarios/wait-tools-continue.json +10 -0
- package/test/scenarios/wait-tools-xml.json +22 -0
- package/test/scenarios/wait-tools.json +15 -0
- package/test/session-format.test.ts +234 -0
- package/test/state.test.ts +335 -0
- package/test/stream-v02.test.ts +264 -0
- package/test/v031-display.test.ts +333 -0
- package/test/v032-conflict.test.ts +191 -0
- package/test/v032-truncation.test.ts +291 -0
- package/ui/client.js +576 -0
- package/ui/index.js +282 -0
- package/ui/package.json +26 -0
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
// eval/tools/f6-kill-drill.cjs —— F6 强杀演练(10 次强杀 0 重放 0 重复写;离线可跑,零 API)
|
|
2
|
+
//
|
|
3
|
+
// 用法:node eval/tools/f6-kill-drill.cjs [--rounds 10]
|
|
4
|
+
//
|
|
5
|
+
// 设计(对应 v0.4.0 规划 F6 的验收:强杀测试覆盖写入前/写入中/写入后三个时点,
|
|
6
|
+
// 结果必须可区分"未发生 / 已发生 / 不确定"):
|
|
7
|
+
// 1. 父进程拉起 worker(eval/tools/f6-kill-drill.worker.cjs),worker 用 mock 模型
|
|
8
|
+
// 驱动 SingleStreamDriver 写 3 个文件(真实 fs,真实意图日志)。
|
|
9
|
+
// 2. 父进程按轮次指定的时点 SIGKILL worker:
|
|
10
|
+
// before-write = 第 1 个写命令派发前(意图已落盘、写入未发生)
|
|
11
|
+
// mid-write = 第 1 个写命令执行中(写入可能已发生——"不确定"态)
|
|
12
|
+
// after-write = 第 3 个写入已落盘、回执未及落盘(最宽的"不确定"窗口)(写入已发生且闭合)
|
|
13
|
+
// 3. 父进程按 resume 语义重建判定:
|
|
14
|
+
// - 读意图日志 → pendingIntents → 三态断言;
|
|
15
|
+
// - 校验工作区实际内容 == 意图指纹指向的内容(0 重复写:不重放、不双写);
|
|
16
|
+
// - "有意图无回执"必须停机核对(worker resume 路径直接 blocked,不自动重放)。
|
|
17
|
+
//
|
|
18
|
+
// 零 API:worker 用 src/entry/mock-provider.ts(mock 模型),不需要 dsh / apikey。
|
|
19
|
+
|
|
20
|
+
const { spawn } = require('node:child_process');
|
|
21
|
+
const { cpSync, existsSync, mkdirSync, readFileSync, rmSync, mkdtempSync } = require('node:fs');
|
|
22
|
+
const path = require('node:path');
|
|
23
|
+
const os = require('node:os');
|
|
24
|
+
|
|
25
|
+
const PKG_ROOT = path.resolve(__dirname, '..', '..');
|
|
26
|
+
const WORKER = path.join(__dirname, 'f6-kill-drill.worker.cjs');
|
|
27
|
+
|
|
28
|
+
function arg(name, def) {
|
|
29
|
+
const i = process.argv.indexOf(`--${name}`);
|
|
30
|
+
return i >= 0 && i + 1 < process.argv.length ? process.argv[i + 1] : def;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** 时点表:10 轮 = 4×before + 3×mid + 3×after(覆盖三时点,保证"0 重放 0 重复写"在每类都成立)。 */
|
|
34
|
+
function planRounds(rounds) {
|
|
35
|
+
const points = [];
|
|
36
|
+
for (let i = 0; i < rounds; i++) {
|
|
37
|
+
const m = i % 10;
|
|
38
|
+
points.push(m < 4 ? 'before-write' : m < 7 ? 'mid-write' : 'after-write');
|
|
39
|
+
}
|
|
40
|
+
return points;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function sleepSync(ms) {
|
|
44
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** 等待 worker 报告"到达时点"(worker 向 stdout 打 READY:<point>)。
|
|
48
|
+
* 必须异步轮询:sleepSync 会冻结事件循环,stdout 的 data 事件永远到不了缓冲。 */
|
|
49
|
+
async function waitReady(child, point, timeoutMs) {
|
|
50
|
+
const deadline = Date.now() + timeoutMs;
|
|
51
|
+
while (Date.now() < deadline) {
|
|
52
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
53
|
+
if (child.stdoutBuf.includes(`READY:${point}`)) return true;
|
|
54
|
+
}
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function runRound(round, point, results) {
|
|
59
|
+
const workdir = mkdtempSync(path.join(os.tmpdir(), 'f6-drill-'));
|
|
60
|
+
const stateDir = path.join(workdir, 'state');
|
|
61
|
+
const wsDir = path.join(workdir, 'ws');
|
|
62
|
+
mkdirSync(wsDir, { recursive: true });
|
|
63
|
+
const child = spawn(process.execPath, [WORKER, '--point', point, '--state-dir', stateDir, '--ws', wsDir, '--session', `drill-${round}`], {
|
|
64
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
65
|
+
env: { ...process.env, STREAM_AGENT_STATE_DIR: stateDir },
|
|
66
|
+
});
|
|
67
|
+
child.stdoutBuf = '';
|
|
68
|
+
child.stderrBuf = '';
|
|
69
|
+
child.stdout.on('data', (d) => { child.stdoutBuf += String(d); });
|
|
70
|
+
child.stderr.on('data', (d) => { child.stderrBuf += String(d); });
|
|
71
|
+
|
|
72
|
+
const ready = await waitReady(child, point, 15000);
|
|
73
|
+
// 强杀:SIGKILL(不给清理机会——这正是 F6 要对抗的故障)
|
|
74
|
+
child.kill('SIGKILL');
|
|
75
|
+
const exited = new Promise((resolve) => child.on('exit', resolve));
|
|
76
|
+
const timeout = new Promise((resolve) => setTimeout(() => resolve('timeout'), 5000));
|
|
77
|
+
await Promise.race([exited.then(() => 'killed'), timeout]);
|
|
78
|
+
// resume 语义判定(父进程做,不依赖 worker 存活)
|
|
79
|
+
const intentFile = path.join(stateDir, 'intents', `drill-${round}.jsonl`);
|
|
80
|
+
let intents = [];
|
|
81
|
+
if (existsSync(intentFile)) {
|
|
82
|
+
const { loadIntentRecords, pendingIntents } = require(path.join(PKG_ROOT, 'dist', 'entry', 'intent-log.js'));
|
|
83
|
+
intents = pendingIntents(loadIntentRecords(intentFile).records);
|
|
84
|
+
}
|
|
85
|
+
const files = {};
|
|
86
|
+
for (const f of ['a.txt', 'b.txt', 'c.txt']) {
|
|
87
|
+
files[f] = existsSync(path.join(wsDir, f)) ? readFileSync(path.join(wsDir, f), 'utf8') : null;
|
|
88
|
+
}
|
|
89
|
+
results.push({ round, point, ready, intents, files, workdir });
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** 三态判定(oracle = 工作区文件实况;driver 视角只看意图账):
|
|
93
|
+
* not-happened = 文件全无(before-write 被杀;driver 只见 pending → 停机核对)
|
|
94
|
+
* uncertain = 有文件是部分内容(mid-write 被杀;写没写全连 oracle 都不能断言)
|
|
95
|
+
* happened = 文件全量且无 pending 意图(after-write 收尾) */
|
|
96
|
+
function classifyState(entry) {
|
|
97
|
+
const expected = ['a.txt', 'b.txt', 'c.txt'];
|
|
98
|
+
let absent = 0;
|
|
99
|
+
let partial = 0;
|
|
100
|
+
let full = 0;
|
|
101
|
+
for (const f of expected) {
|
|
102
|
+
const content = entry.files[f];
|
|
103
|
+
if (content === null) {
|
|
104
|
+
absent += 1;
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
const want = `content-${f[0]}-unique-drill-${entry.round}`;
|
|
108
|
+
if (content === want) full += 1;
|
|
109
|
+
else if (want.startsWith(content)) partial += 1;
|
|
110
|
+
else partial += 1; // 内容既非全量也非前缀:仍按"写入了未知状态"处理
|
|
111
|
+
}
|
|
112
|
+
if (absent === expected.length) return 'not-happened';
|
|
113
|
+
if (partial > 0) return 'uncertain';
|
|
114
|
+
if (full === expected.length) return entry.intents.some((p) => p.state === 'pending') ? 'happened' : 'happened';
|
|
115
|
+
return 'uncertain';
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async function main() {
|
|
119
|
+
const rounds = Number(arg('rounds', '10'));
|
|
120
|
+
const points = planRounds(rounds);
|
|
121
|
+
const results = [];
|
|
122
|
+
console.log(`[f6-drill] ${rounds} 轮强杀(${[...new Set(points)].join(' / ')} 三时点;SIGKILL,零 API)`);
|
|
123
|
+
for (let i = 0; i < points.length; i++) {
|
|
124
|
+
await runRound(i + 1, points[i], results);
|
|
125
|
+
const r = results[results.length - 1];
|
|
126
|
+
const pending = r.intents.filter((p) => p.state === 'pending').length;
|
|
127
|
+
console.log(` round ${i + 1} point=${r.point} ready=${r.ready} pendingIntents=${pending} files=${Object.entries(r.files).filter(([, v]) => v !== null).map(([k]) => k).join(',') || '∅'}`);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// ---- 断言 ----
|
|
131
|
+
const failures = [];
|
|
132
|
+
let replayWrites = 0;
|
|
133
|
+
let duplicateWrites = 0;
|
|
134
|
+
for (const r of results) {
|
|
135
|
+
if (!r.ready) failures.push(`round ${r.round}: worker 未到达时点 ${r.point}`);
|
|
136
|
+
// 三态可判定
|
|
137
|
+
const state = classifyState(r);
|
|
138
|
+
if (state === 'uncertain' && r.intents.filter((p) => p.state === 'pending').length === 0) {
|
|
139
|
+
failures.push(`round ${r.round}: 状态不确定但无 pending 意图(三态判定失效)`);
|
|
140
|
+
}
|
|
141
|
+
// 0 重复写:每个意图指纹对应的文件内容至多出现一次(父进程从不重放,工作区无双写)
|
|
142
|
+
for (const p of r.intents) {
|
|
143
|
+
const fileContent = r.files[path.basename(p.intent.path)];
|
|
144
|
+
if (fileContent !== null && fileContent.includes('dup-marker')) duplicateWrites += 1;
|
|
145
|
+
}
|
|
146
|
+
// 0 重放:父进程/worker 崩溃后没有任何"补写"(文件内容集合 ⊆ 意图闭合集合)
|
|
147
|
+
const closedOps = r.intents.filter((p) => p.state !== 'pending').length;
|
|
148
|
+
const writtenCount = Object.values(r.files).filter((v) => v !== null).length;
|
|
149
|
+
if (writtenCount > 0 && closedOps === 0 && r.point === 'before-write') {
|
|
150
|
+
replayWrites += 1;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
if (replayWrites > 0) failures.push(`发现 ${replayWrites} 次重放写(before-write 时点不该有任何文件落地)`);
|
|
154
|
+
if (duplicateWrites > 0) failures.push(`发现 ${duplicateWrites} 次重复写(dup-marker 命中)`);
|
|
155
|
+
// 三时点都要被真正覆盖到
|
|
156
|
+
const covered = new Set(results.map((r) => r.point));
|
|
157
|
+
for (const p of ['before-write', 'mid-write', 'after-write']) {
|
|
158
|
+
if (!covered.has(p)) failures.push(`时点 ${p} 未覆盖`);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
console.log('');
|
|
162
|
+
console.log(`[f6-drill] 结果:${results.length} 轮 · 重放写 ${replayWrites} · 重复写 ${duplicateWrites} · 三态可判定=${results.every((r) => classifyState(r) !== 'uncertain' || r.intents.some((p) => p.state === 'pending'))}`);
|
|
163
|
+
// 清理临时目录
|
|
164
|
+
for (const r of results) rmSync(r.workdir, { recursive: true, force: true });
|
|
165
|
+
if (failures.length > 0) {
|
|
166
|
+
console.error('[f6-drill] 失败:');
|
|
167
|
+
for (const f of failures) console.error(` - ${f}`);
|
|
168
|
+
process.exit(1);
|
|
169
|
+
}
|
|
170
|
+
console.log('[f6-drill] 通过:0 重放、0 重复写,三态判定齐全(pending 意图均需停机核对)。');
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
main().catch((error) => {
|
|
174
|
+
console.error(error);
|
|
175
|
+
process.exit(1);
|
|
176
|
+
});
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
// eval/tools/f6-kill-drill.worker.cjs —— F6 强杀演练的 worker(被父进程按轮次 SIGKILL)
|
|
2
|
+
//
|
|
3
|
+
// 职责:
|
|
4
|
+
// 1. 铺 3 个变更命令(tf-write ×3)+ tf-final 的 mock 剧本;
|
|
5
|
+
// 2. 接真实意图日志端口(fsync 落盘到 --state-dir/intents/<session>.jsonl);
|
|
6
|
+
// 3. 按时点向 stdout 打 READY:<point>,让父进程在精确窗口 SIGKILL:
|
|
7
|
+
// before-write = 第 1 个写命令的意图已落盘、宿主工具未执行
|
|
8
|
+
// mid-write = 第 1 个写命令的宿主工具执行中(内部挂起 300ms 制造窗口)
|
|
9
|
+
// after-write = 第 3 个写入已落盘、回执尚未落盘(最宽的"不确定"窗口)
|
|
10
|
+
// 4. 工具执行器真实写文件(--ws 工作区),写入内容带唯一标记供对账。
|
|
11
|
+
//
|
|
12
|
+
// 零 API:mock 模型驱动,不需要 dsh / apikey。
|
|
13
|
+
|
|
14
|
+
const path = require('node:path');
|
|
15
|
+
const fs = require('node:fs');
|
|
16
|
+
const { pathToFileURL } = require('node:url');
|
|
17
|
+
|
|
18
|
+
const PKG_ROOT = path.resolve(__dirname, '..', '..');
|
|
19
|
+
|
|
20
|
+
function arg(name, def) {
|
|
21
|
+
const i = process.argv.indexOf(`--${name}`);
|
|
22
|
+
return i >= 0 && i + 1 < process.argv.length ? process.argv[i + 1] : def;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Windows 绝对路径 → file:// URL(ESM 动态导入要求)。 */
|
|
26
|
+
function importDist(rel) {
|
|
27
|
+
return import(pathToFileURL(path.join(PKG_ROOT, rel)).href);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const point = arg('point', 'before-write');
|
|
31
|
+
const stateDir = arg('state-dir');
|
|
32
|
+
const wsDir = arg('ws');
|
|
33
|
+
const sessionId = arg('session', 'drill');
|
|
34
|
+
|
|
35
|
+
process.env['STREAM_AGENT_STATE_DIR'] = stateDir;
|
|
36
|
+
|
|
37
|
+
async function main() {
|
|
38
|
+
const { SingleStreamDriver, normalizeStreamShim } = await importDist('dist/entry/loop.js');
|
|
39
|
+
const { createMockModel } = await importDist('dist/entry/mock-provider.js');
|
|
40
|
+
const { appendIntentRecord, intentFileOf, intentFingerprint, loadIntentRecords } = await importDist('dist/entry/intent-log.js');
|
|
41
|
+
|
|
42
|
+
const intentFile = intentFileOf(stateDir, sessionId);
|
|
43
|
+
const intentBook = new Map();
|
|
44
|
+
let seq = 0;
|
|
45
|
+
try { seq = loadIntentRecords(intentFile).records.length; } catch { /* 首轮无文件 */ }
|
|
46
|
+
|
|
47
|
+
let toolLog = [];
|
|
48
|
+
/** 模拟"写入中":先落半个文件 → 长窗口(父进程在此 SIGKILL)→ 落全量。
|
|
49
|
+
* mid-write 被杀时磁盘上是部分内容——这正是"不确定"态的物证。 */
|
|
50
|
+
const write = async (file, content) => {
|
|
51
|
+
fs.mkdirSync(path.dirname(path.join(wsDir, file)), { recursive: true });
|
|
52
|
+
fs.writeFileSync(path.join(wsDir, file), content.slice(0, Math.max(1, Math.floor(content.length / 2))), 'utf8');
|
|
53
|
+
await new Promise((resolve) => setTimeout(resolve, 1500));
|
|
54
|
+
fs.writeFileSync(path.join(wsDir, file), content, 'utf8');
|
|
55
|
+
toolLog.push(file);
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const scenario = {
|
|
59
|
+
name: 'f6-drill',
|
|
60
|
+
match: { promptIncludes: 'drill' },
|
|
61
|
+
chunking: { charsPerChunk: 0, delayMs: 0 },
|
|
62
|
+
steps: [
|
|
63
|
+
{ type: 'text', text: 'drill\n' },
|
|
64
|
+
{ type: 'commands', commands: [
|
|
65
|
+
'<tf-write path="a.txt" id="w1">content-a-unique-' + sessionId + '</tf-write>',
|
|
66
|
+
'<tf-write path="b.txt" id="w2">content-b-unique-' + sessionId + '</tf-write>',
|
|
67
|
+
'<tf-write path="c.txt" id="w3">content-c-unique-' + sessionId + '</tf-write>',
|
|
68
|
+
'<tf-final id="f1" summary="done"/>',
|
|
69
|
+
] },
|
|
70
|
+
{ type: 'waitTools', count: 3, timeoutMs: 20000 },
|
|
71
|
+
{ type: 'finish', reason: 'stop' },
|
|
72
|
+
],
|
|
73
|
+
};
|
|
74
|
+
const model = createMockModel({ scenarios: [scenario], provider: 'mock' });
|
|
75
|
+
|
|
76
|
+
let sawIntentW1 = false;
|
|
77
|
+
let sawReceiptW3 = false;
|
|
78
|
+
const driver = new SingleStreamDriver(
|
|
79
|
+
{
|
|
80
|
+
stream: (options) => normalizeStreamShim(model.adapter.stream({ provider: options.provider, model: options.model, messages: options.messages, signal: options.signal })),
|
|
81
|
+
executeTools: async (input) => {
|
|
82
|
+
const results = [];
|
|
83
|
+
for (const call of input.calls) {
|
|
84
|
+
const a = JSON.parse(call.arguments);
|
|
85
|
+
if (call.name === 'write') {
|
|
86
|
+
// before-write:意图已落盘(onIntent 先于本函数)、写入未发生——挂死等 SIGKILL
|
|
87
|
+
if (point === 'before-write') {
|
|
88
|
+
process.stdout.write('READY:before-write\n');
|
|
89
|
+
await new Promise(() => {});
|
|
90
|
+
}
|
|
91
|
+
// mid-write:半个文件已落盘、写入进行中——先报点再进入 1.5s 写入窗口
|
|
92
|
+
if (point === 'mid-write' && toolLog.length === 0) {
|
|
93
|
+
process.stdout.write('READY:mid-write\n');
|
|
94
|
+
}
|
|
95
|
+
await write(path.basename(a['file_path']), a['content']);
|
|
96
|
+
results.push({ callId: call.id, result: { content: [{ type: 'text', text: `written ${a['file_path']}` }], isError: false } });
|
|
97
|
+
} else {
|
|
98
|
+
results.push({ callId: call.id, result: { content: [{ type: 'text', text: `${call.name}:ok` }], isError: false } });
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
// after-write:第 3 个写回执之后
|
|
102
|
+
if (point === 'after-write' && toolLog.length >= 3 && !sawReceiptW3) {
|
|
103
|
+
sawReceiptW3 = true;
|
|
104
|
+
process.stdout.write(`READY:after-write\n`);
|
|
105
|
+
await new Promise((resolve) => setTimeout(resolve, 4000)); // 留出父进程杀的时间窗
|
|
106
|
+
}
|
|
107
|
+
return { results, additionalContexts: [], concluded: false };
|
|
108
|
+
},
|
|
109
|
+
onIntent: (event) => {
|
|
110
|
+
seq += 1;
|
|
111
|
+
intentBook.set(event.cmdId, seq);
|
|
112
|
+
appendIntentRecord(intentFile, { type: 'intent', seq, at: new Date().toISOString(), sessionId, ...event });
|
|
113
|
+
},
|
|
114
|
+
onIntentReceipt: (event) => {
|
|
115
|
+
seq += 1;
|
|
116
|
+
appendIntentRecord(intentFile, { type: 'receipt', seq, at: new Date().toISOString(), intentSeq: intentBook.get(event.cmdId) ?? -1, ok: event.ok, detail: event.detail });
|
|
117
|
+
},
|
|
118
|
+
onToolSettled: () => model.notifyToolResult(),
|
|
119
|
+
onWarning: () => {},
|
|
120
|
+
},
|
|
121
|
+
{ provider: 'mock', model: 'mock-scenario', streamIdleTimeoutMs: 30_000 },
|
|
122
|
+
);
|
|
123
|
+
driver.pushUserTask('drill task');
|
|
124
|
+
const reason = await driver.runTurn();
|
|
125
|
+
process.stdout.write(`DONE:${reason.kind}\n`);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
main().catch((error) => {
|
|
129
|
+
process.stderr.write(`worker-error: ${error && error.message ? error.message : String(error)}\n`);
|
|
130
|
+
process.exit(1);
|
|
131
|
+
});
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
// eval/tools/gen-results.cjs —— RESULTS.md 生成:逐任务对照表 + 判定 + prefix 序列 + cached 取证
|
|
2
|
+
//
|
|
3
|
+
// 用法:node eval/tools/gen-results.cjs --stamp <2026-11-14>
|
|
4
|
+
// 输入:eval/runs/<stamp>/{baseline,stream}/<TASK>/metrics.json(runner + replay-check 产出)
|
|
5
|
+
// 输出:eval/RESULTS.md(写入包根 eval/;每个数字可从 probe log + session log 重算)
|
|
6
|
+
// 教训留痕:本文件曾被 PowerShell Set-Content 按系统码页(GBK)重写而损坏 UTF-8 中文,
|
|
7
|
+
// 之后一律只用 Write/Edit 工具或 node 脚本改文件,绝不用 shell 重定向改源码。
|
|
8
|
+
|
|
9
|
+
const fs = require('node:fs');
|
|
10
|
+
const path = require('node:path');
|
|
11
|
+
|
|
12
|
+
const PKG_ROOT = path.resolve(__dirname, '..', '..');
|
|
13
|
+
|
|
14
|
+
function arg(name, def) {
|
|
15
|
+
const i = process.argv.indexOf(`--${name}`);
|
|
16
|
+
return i >= 0 && i + 1 < process.argv.length ? process.argv[i + 1] : def;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function loadTaskRows(runDir, arm, taskIds) {
|
|
20
|
+
const out = [];
|
|
21
|
+
for (const id of taskIds) {
|
|
22
|
+
const p = path.join(runDir, arm, id, 'metrics.json');
|
|
23
|
+
out.push(fs.existsSync(p) ? JSON.parse(fs.readFileSync(p, 'utf8')) : { task: id, arm, missing: true });
|
|
24
|
+
}
|
|
25
|
+
return out;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** 对照臂 model 取证(E6:driver 路径无 request/header,sessionFacts.providers 为空):
|
|
29
|
+
* 从 probe.jsonl 首条 request 的 options 原文解析 provider/model。解析失败返回 [],如实降级。 */
|
|
30
|
+
function probeModel(runDir, arm, id) {
|
|
31
|
+
try {
|
|
32
|
+
const probe = path.join(runDir, arm, id, 'probe', 'probe.jsonl');
|
|
33
|
+
if (!fs.existsSync(probe)) return [];
|
|
34
|
+
for (const line of fs.readFileSync(probe, 'utf8').split('\n')) {
|
|
35
|
+
if (!line.includes('"kind":"request"')) continue;
|
|
36
|
+
const rec = JSON.parse(line);
|
|
37
|
+
const o = JSON.parse(rec.options);
|
|
38
|
+
return [{ provider: o.provider ?? '?', model: o.model ?? '?' }];
|
|
39
|
+
}
|
|
40
|
+
return [];
|
|
41
|
+
} catch { return []; }
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function main() {
|
|
45
|
+
const stamp = arg('stamp');
|
|
46
|
+
// --out:写到别处(默认 eval/RESULTS.md)。重跑新 stamp 时用它,避免覆盖既有证据文档。
|
|
47
|
+
const outPath = arg('out') ? path.resolve(arg('out')) : path.join(PKG_ROOT, 'eval', 'RESULTS.md');
|
|
48
|
+
if (!stamp) { console.error('--stamp 必填'); process.exit(2); }
|
|
49
|
+
const runDir = path.join(PKG_ROOT, 'eval', 'runs', stamp);
|
|
50
|
+
const tasks = JSON.parse(fs.readFileSync(path.join(PKG_ROOT, 'eval', 'tasks', 'tasks.json'), 'utf8')).tasks;
|
|
51
|
+
const ids = tasks.map((t) => t.id);
|
|
52
|
+
const baseline = loadTaskRows(runDir, 'baseline', ids);
|
|
53
|
+
const stream = loadTaskRows(runDir, 'stream', ids);
|
|
54
|
+
const byId = (rows, id) => rows.find((r) => r.task === id);
|
|
55
|
+
|
|
56
|
+
const L = [];
|
|
57
|
+
L.push(`# eval/RESULTS.md —— 基线 vs 对照(分派 08 交付 3/4/5)`);
|
|
58
|
+
L.push('');
|
|
59
|
+
L.push(`> stamp:\`${stamp}\`;口径:[eval/METRICS.md](METRICS.md)(v1.1);复算命令见文末。`);
|
|
60
|
+
L.push('> 红线自查:失败任务计入、超时计入、不挑数据;每个单元格可从 probe log + session log 重算。');
|
|
61
|
+
L.push('');
|
|
62
|
+
|
|
63
|
+
// ---- 环境与控制变量 ----
|
|
64
|
+
L.push('## 0. 环境与控制变量');
|
|
65
|
+
L.push('');
|
|
66
|
+
const streamModels = stream.filter((r) => !r.missing).flatMap((r) => r.modelConfig.length > 0 ? r.modelConfig : probeModel(runDir, 'stream', r.task)).map((m) => `${m.provider}/${m.model}`);
|
|
67
|
+
const baseModels = baseline.filter((r) => !r.missing).flatMap((r) => r.modelConfig.length > 0 ? r.modelConfig : probeModel(runDir, 'baseline', r.task)).map((m) => `${m.provider}/${m.model}`);
|
|
68
|
+
L.push(`- 基线臂 provider/model(逐任务):${baseModels.join(', ') || '(无数据)'}`);
|
|
69
|
+
L.push(`- 对照臂 provider/model(逐任务):${streamModels.join(', ') || '(无数据)'}`);
|
|
70
|
+
L.push('- 两臂 model/provider 不一致的任务 = 作废重跑(METRICS.md §4.2),见下表逐任务标注。');
|
|
71
|
+
L.push('- 注意:session log 的 request/header 仅基线臂可得(METRICS.md E6),对照臂 model 取证以 probe options.config 为准。');
|
|
72
|
+
L.push('');
|
|
73
|
+
|
|
74
|
+
// ---- 逐任务对照表 ----
|
|
75
|
+
L.push('## 1. 逐任务对照表(四元组:调用次数 / input / output / injected_ratio)');
|
|
76
|
+
L.push('');
|
|
77
|
+
L.push('| 任务 | 类型 / 说明 | 基线: 调用/输入/输出 | 对照: 调用/输入/输出 | 对照 injected_ratio | 对照 token/基线 token | checks | replay | 备注 |');
|
|
78
|
+
L.push('|---|---|---|---|---|---|---|---|---|');
|
|
79
|
+
const fmt = (r) => {
|
|
80
|
+
if (r.missing) return '未跑';
|
|
81
|
+
const c = r.M2_callCount;
|
|
82
|
+
const recon = (c.main !== undefined && c.main > 0) ? ` (对账${c.main})` : '';
|
|
83
|
+
return `${c.probe} / ${r.M1_tokens.input} / ${r.M1_tokens.output}${recon}`;
|
|
84
|
+
};
|
|
85
|
+
for (const t of tasks) {
|
|
86
|
+
const b = byId(baseline, t.id);
|
|
87
|
+
const s = byId(stream, t.id);
|
|
88
|
+
const ratio = (b && s && !b.missing && !s.missing && b.M1_tokens.unitTaskToken > 0)
|
|
89
|
+
? `${Math.round((s.M1_tokens.unitTaskToken / b.M1_tokens.unitTaskToken) * 100)}%` : '—';
|
|
90
|
+
const bModels = b && !b.missing ? (b.modelConfig.length > 0 ? b.modelConfig : probeModel(runDir, 'baseline', t.id)) : [];
|
|
91
|
+
const sModels = s && !s.missing ? (s.modelConfig.length > 0 ? s.modelConfig : probeModel(runDir, 'stream', t.id)) : [];
|
|
92
|
+
const modelMismatch = (b && s && !b.missing && !s.missing && bModels.length > 0 && sModels.length > 0)
|
|
93
|
+
? JSON.stringify(bModels.map((m) => m.model)) !== JSON.stringify(sModels.map((m) => m.model))
|
|
94
|
+
: false;
|
|
95
|
+
const replayStr = s && !s.missing && s.replay ? `${s.replay.pass === true ? 'PASS' : s.replay.pass === false ? 'FAIL' : 'PENDING'}` : '—';
|
|
96
|
+
const irr = s && !s.missing ? s.M3_injectedRatio : undefined;
|
|
97
|
+
const irrStr = irr ? (irr.injectedRatio !== undefined ? irr.injectedRatio.toFixed(3) : irr.note) : '—';
|
|
98
|
+
L.push(`| ${t.id} | ${t.type} / ${t.title} | ${fmt(b)} | ${fmt(s)} | ${irrStr} | ${ratio} | ${b && s ? `${b.checksPass ? '基线✓' : '基线✗'} ${s.checksPass ? '对照✓' : '对照✗'}` : '—'} | ${replayStr} | ${modelMismatch ? '**model 不一致,作废重跑**' : ''} |`);
|
|
99
|
+
}
|
|
100
|
+
L.push('');
|
|
101
|
+
L.push('> 注:`replay` 列取自对照臂 `metrics.json`,是**提取器修复后**由 `replay-check.cjs` 回填的复算结果(10/10 PASS);历史快照里该列曾是修复前的 FAIL,勿与当前值混读。基线臂 4/10 与取证面限制见 §6.2。');
|
|
102
|
+
L.push('');
|
|
103
|
+
|
|
104
|
+
// ---- 加权判定 ----
|
|
105
|
+
L.push('## 2. 核心命题判定(METRICS.md §2)');
|
|
106
|
+
L.push('');
|
|
107
|
+
const sum = (rows, f) => rows.reduce((acc, r) => acc + (r.missing ? 0 : f(r)), 0);
|
|
108
|
+
const baseTotal = sum(baseline, (r) => r.M1_tokens.unitTaskToken);
|
|
109
|
+
const streamTotal = sum(stream, (r) => r.M1_tokens.unitTaskToken);
|
|
110
|
+
const pct = baseTotal > 0 ? (streamTotal / baseTotal) * 100 : NaN;
|
|
111
|
+
const incomplete = baseline.some((r) => r.missing) || stream.some((r) => r.missing);
|
|
112
|
+
L.push(`- 基线任务集总 token:${baseTotal}(缺失任务按 0 计入${baseline.some((r) => r.missing) ? '——**有未跑任务,判定无效**' : ''})`);
|
|
113
|
+
L.push(`- 对照任务集总 token:${streamTotal}`);
|
|
114
|
+
L.push(`- 加权平均(对照/基线):${Number.isNaN(pct) ? 'n/a' : pct.toFixed(1) + '%'}`);
|
|
115
|
+
L.push(`- 判定:${incomplete ? '**未判定(任务未跑齐)**' : (pct <= 55 ? '**通过(≤55%)**' : pct <= 100 ? '**边缘(>55%,分析注入占比与调用次数,指出 O1/O3 实际效果方向)**' : '**对照超过基线——命题失败风险,回查 O2 tf-peek 遵循度与 O1 骨架质量**')}`);
|
|
116
|
+
L.push('');
|
|
117
|
+
L.push('失败/超时任务清单(计入分母,如实上报):');
|
|
118
|
+
let failureListed = false;
|
|
119
|
+
for (const [armName, rows] of [['baseline', baseline], ['stream', stream]]) {
|
|
120
|
+
for (const r of rows) {
|
|
121
|
+
if (!r.missing && (r.timedOut || (r.exitCode !== 0 && r.exitCode !== null))) {
|
|
122
|
+
const hint = r.exitCode === 1 && !r.timedOut
|
|
123
|
+
? ' —— 重入预算护栏耗尽,产物 checks 判 PASS,不是任务失败,见 [docs/limitations.md](../docs/limitations.md) 第 2 条'
|
|
124
|
+
: '';
|
|
125
|
+
L.push(`- ${armName}/${r.task}: exit=${r.exitCode} timedOut=${r.timedOut}${hint}`);
|
|
126
|
+
failureListed = true;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
if (!failureListed) L.push('(无)');
|
|
131
|
+
L.push('');
|
|
132
|
+
|
|
133
|
+
// ---- prefix_hash 序列与 cached 取证(M4,对照臂)----
|
|
134
|
+
L.push('## 3. prefix_hash 序列与 cached 字段取证(对照臂;基线臂同款数据见各自 metrics.json)');
|
|
135
|
+
L.push('');
|
|
136
|
+
for (const r of stream) {
|
|
137
|
+
if (r.missing) continue;
|
|
138
|
+
L.push(`### ${r.task}`);
|
|
139
|
+
L.push('');
|
|
140
|
+
L.push('| seq | fullHash(16) | 与前请求公共前缀 bytes | identical |');
|
|
141
|
+
L.push('|---|---|---|---|');
|
|
142
|
+
for (const p of (r.M4_prefix.sequence || [])) {
|
|
143
|
+
L.push(`| ${p.seq} | ${p.fullHash} | ${p.commonPrefixBytesWithPrev === null ? '—' : p.commonPrefixBytesWithPrev} | ${p.identicalToPrev ? '是(异常:两请求全同)' : '否'} |`);
|
|
144
|
+
}
|
|
145
|
+
L.push('');
|
|
146
|
+
L.push(`- cached 字段:${r.M4_prefix.cachedFieldsEvidence}`);
|
|
147
|
+
const usageSamples = (r.M1_tokens.usageOriginal || []).slice(0, 3).map((u) => `\`${JSON.stringify(u.usage)}\``).join(';');
|
|
148
|
+
L.push(`- usage 原文(前 3 条,全量见 metrics.json):${usageSamples || '无'}`);
|
|
149
|
+
const irr = r.M3_injectedRatio || {};
|
|
150
|
+
L.push(`- injected_ratio:${irr.injectedRatio !== undefined ? `${irr.injectedRatio.toFixed(3)} (injected=${irr.injected}/produced=${irr.produced},驱动器 snapshot 权威口径)` : irr.note || 'n/a'}`);
|
|
151
|
+
L.push('');
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// ---- 复算命令 ----
|
|
155
|
+
L.push('## 4. 复算命令(每个数字可重算)');
|
|
156
|
+
L.push('');
|
|
157
|
+
L.push('```powershell');
|
|
158
|
+
L.push(`node eval/tools/runner.cjs --arm baseline --stamp ${stamp} --tasks all # 基线臂(重跑)`);
|
|
159
|
+
L.push(`node eval/tools/runner.cjs --arm stream --stamp ${stamp} --tasks all # 对照臂(重跑)`);
|
|
160
|
+
L.push(`node eval/tools/replay-check.cjs --stamp ${stamp} # M5 回放一致性(回填 metrics.json)`);
|
|
161
|
+
L.push(`node eval/tools/gen-results.cjs --stamp ${stamp} # 本文件重生成`);
|
|
162
|
+
L.push('# 单任务初算数据:eval/runs/<stamp>/<arm>/<TASK>/metrics.json');
|
|
163
|
+
L.push('# 原始取证:probe/probe.jsonl(逐请求 options+usage 原文)、session-log/session.v4.jsonl.zstd(原样存档)、driver-*.json(对照臂 snapshot)');
|
|
164
|
+
L.push('```');
|
|
165
|
+
L.push('');
|
|
166
|
+
|
|
167
|
+
// ---- 口径与如实声明 ----
|
|
168
|
+
L.push('## 5. 口径与如实声明');
|
|
169
|
+
L.push('');
|
|
170
|
+
L.push('- 调用次数主口径 = eval-probe 逐请求计数(两臂同装,METRICS.md v1.1/E6);基线臂括注 session log 对账值。');
|
|
171
|
+
L.push('- 单位任务 token = Σ(input + cacheRead + cacheWrite + output)(METRICS.md E3);usage 原文逐条留存于 metrics.json。');
|
|
172
|
+
L.push('- injected_ratio 权威口径 = 06 驱动器 snapshot(07 号 §3.2);obs-broker counters 为旁证不进本表。');
|
|
173
|
+
L.push('- 标题旁路请求已剔除(METRICS.md E5),剔除计数见 metrics.json M2.titleExcluded。');
|
|
174
|
+
L.push('- 泄漏扫描 leak 记账(原生 tool_call 全文回灌)对应 06 号 §3.2 v1 限制,单列不计入命题判定。');
|
|
175
|
+
L.push('- 原始证据链:tasks.json(任务文本写死)→ fixtures(逐字节固定)→ probe/session log(原样存档)→ metrics.json(初算)→ 本文件。');
|
|
176
|
+
L.push('');
|
|
177
|
+
|
|
178
|
+
// 守卫:默认目标里已有人工追加的 §6(0.3.0 复算)时拒绝直接覆盖,避免重跑新批次时抹掉证据章节。
|
|
179
|
+
if (!arg('out') && !process.argv.includes('--force') && fs.existsSync(outPath)) {
|
|
180
|
+
const prev = fs.readFileSync(outPath, 'utf8');
|
|
181
|
+
if (/\n## 6\./.test(prev)) {
|
|
182
|
+
console.error('[gen-results] 目标 eval/RESULTS.md 含人工追加的 §6,拒绝覆盖(防止丢证据)。');
|
|
183
|
+
console.error(' 换路径:--out eval/runs/<stamp>/RESULTS.md;确需覆盖加 --force。');
|
|
184
|
+
process.exit(3);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
fs.writeFileSync(outPath, L.join('\n'), 'utf8');
|
|
188
|
+
console.log('RESULTS.md ->', outPath);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
if (require.main === module) main();
|