@hugesb/dsh-stream-agent 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +497 -0
- package/CONTRIBUTING.md +72 -0
- package/LICENSE +202 -0
- package/README.en.md +349 -0
- package/README.md +300 -3
- package/cordis.patch.yml +77 -0
- package/dist/entry/action-gate.js +206 -0
- package/dist/entry/adapter-kit.js +192 -0
- package/dist/entry/context-render.js +119 -0
- package/dist/entry/control-bar-cli.js +289 -0
- package/dist/entry/control-bar.js +123 -0
- package/dist/entry/control-session.js +115 -0
- package/dist/entry/control-state.js +125 -0
- package/dist/entry/cordis-compat.js +95 -0
- package/dist/entry/eval-probe.js +112 -0
- package/dist/entry/intent-log.js +121 -0
- package/dist/entry/json-persistence.js +306 -0
- package/dist/entry/loop.js +3852 -0
- package/dist/entry/mock-provider.js +536 -0
- package/dist/entry/obs-broker.js +178 -0
- package/dist/entry/ptc-bridge.js +405 -0
- package/dist/entry/state-handle.js +217 -0
- package/dist/entry/takeover-registry.js +60 -0
- package/dist/kernel/control-bar.js +559 -0
- package/dist/kernel/final.js +62 -0
- package/dist/kernel/gate.js +464 -0
- package/dist/kernel/modes.js +253 -0
- package/dist/kernel/obs.js +517 -0
- package/dist/kernel/parser.js +632 -0
- package/dist/kernel/render.js +444 -0
- package/dist/kernel/state.js +416 -0
- package/dist/version.js +9 -0
- package/docs/TEST-PROPOSITIONS.md +139 -0
- package/docs/adr-001-segment-boundary.md +61 -0
- package/docs/architecture.md +104 -0
- package/docs/desktop-ui-feasibility.md +277 -0
- package/docs/disabled-lines-audit.md +53 -0
- package/docs/limitations.md +181 -0
- package/docs/protocol.md +99 -0
- package/docs/v0.4.0-plan.md +50 -0
- package/eval/METRICS.md +128 -0
- package/eval/RESULTS.md +246 -0
- package/eval/tasks/fixtures/big-report.md +298 -0
- package/eval/tasks/fixtures/build-manifest.txt +5 -0
- package/eval/tasks/fixtures/deploy.yaml +4 -0
- package/eval/tasks/fixtures/error.log +200 -0
- package/eval/tasks/fixtures/features.yaml +4 -0
- package/eval/tasks/fixtures/pipeline.yaml +4 -0
- package/eval/tasks/fixtures/sales-q1.csv +61 -0
- package/eval/tasks/fixtures/settings.yaml +4 -0
- package/eval/tasks/fixtures/users-roster.json +304 -0
- package/eval/tasks/fixtures/users-scores.json +20 -0
- package/eval/tasks/gen-fixtures.cjs +80 -0
- package/eval/tasks/tasks.json +150 -0
- package/eval/tools/f6-kill-drill.cjs +176 -0
- package/eval/tools/f6-kill-drill.worker.cjs +131 -0
- package/eval/tools/gen-results.cjs +191 -0
- package/eval/tools/machine-test.cjs +200 -0
- package/eval/tools/manual-code-scenarios.md +59 -0
- package/eval/tools/replay-check.cjs +311 -0
- package/eval/tools/runner.cjs +219 -0
- package/eval/tools/session-log.cjs +106 -0
- package/eval/tools/summarize.cjs +185 -0
- package/eval/tools/t3-cells.cjs +173 -0
- package/package.json +73 -6
- package/patches/adapters-smoke.yml +41 -0
- package/patches/eval-account.yml +36 -0
- package/patches/eval-baseline.yml +33 -0
- package/patches/eval-stream-code-off.yml +46 -0
- package/patches/eval-stream-code-on.yml +47 -0
- package/patches/eval-stream.yml +54 -0
- package/patches/takeover.yml +42 -0
- package/patches/test-preset.yml +23 -0
- package/preset/stream/cordis.patch.yml +39 -0
- package/preset/stream/package.json +13 -0
- package/preset.yml +56 -0
- package/profiles/stream/README.md +66 -0
- package/profiles/stream/cordis.patch.yml +12 -0
- package/profiles/stream/package.json +17 -0
- package/profiles/stream-headless/package.json +16 -0
- package/scripts/build.mjs +103 -0
- package/scripts/check-consistency.mjs +146 -0
- package/scripts/gate.mjs +138 -0
- package/scripts/install-check.mjs +116 -0
- package/scripts/install-cycle.mjs +88 -0
- package/scripts/release.mjs +415 -0
- package/scripts/repair-session-logs.mjs +403 -0
- package/src/entry/action-gate.ts +206 -0
- package/src/entry/adapter-kit.ts +192 -0
- package/src/entry/context-render.ts +119 -0
- package/src/entry/control-bar-cli.ts +289 -0
- package/src/entry/control-bar.ts +123 -0
- package/src/entry/control-session.ts +115 -0
- package/src/entry/control-state.ts +125 -0
- package/src/entry/cordis-compat.ts +95 -0
- package/src/entry/eval-probe.ts +112 -0
- package/src/entry/intent-log.ts +121 -0
- package/src/entry/json-persistence.ts +306 -0
- package/src/entry/loop.ts +3852 -0
- package/src/entry/mock-provider.ts +536 -0
- package/src/entry/obs-broker.ts +178 -0
- package/src/entry/ptc-bridge.ts +405 -0
- package/src/entry/state-handle.ts +217 -0
- package/src/entry/takeover-registry.ts +60 -0
- package/src/kernel/control-bar.ts +559 -0
- package/src/kernel/final.ts +62 -0
- package/src/kernel/gate.ts +464 -0
- package/src/kernel/modes.ts +253 -0
- package/src/kernel/obs.ts +517 -0
- package/src/kernel/parser.ts +632 -0
- package/src/kernel/render.ts +444 -0
- package/src/kernel/state.ts +416 -0
- package/src/version.ts +9 -0
- package/test/activation.test.ts +187 -0
- package/test/adapters.test.ts +676 -0
- package/test/control-bar.test.ts +486 -0
- package/test/desktop-ui.test.ts +359 -0
- package/test/eval-probe.test.ts +139 -0
- package/test/f5-plan.test.ts +221 -0
- package/test/f6-intent.test.ts +168 -0
- package/test/f7-segment.test.ts +190 -0
- package/test/fix-limits.test.ts +599 -0
- package/test/gate.test.ts +380 -0
- package/test/helpers/stub-loop.ts +240 -0
- package/test/kernel/obs.test.ts +329 -0
- package/test/kernel/parser.test.ts +450 -0
- package/test/loop.test.ts +1393 -0
- package/test/mock-provider.test.ts +386 -0
- package/test/ptc-bridge.test.ts +356 -0
- package/test/render.test.ts +418 -0
- package/test/s2-install-check.test.ts +46 -0
- package/test/scenarios/3-file-basic.json +13 -0
- package/test/scenarios/5-file-basic.json +22 -0
- package/test/scenarios/burn-more.json +9 -0
- package/test/scenarios/degraded-final.json +10 -0
- package/test/scenarios/fault-abort-xml.json +11 -0
- package/test/scenarios/fault-malformed-command.json +12 -0
- package/test/scenarios/fault-midstream-abort.json +11 -0
- package/test/scenarios/fault-saturating-output.json +9 -0
- package/test/scenarios/fault-timeout.json +9 -0
- package/test/scenarios/peek-first.json +11 -0
- package/test/scenarios/peek-second.json +11 -0
- package/test/scenarios/peek-third.json +10 -0
- package/test/scenarios/recover-final.json +10 -0
- package/test/scenarios/reroute-first.json +17 -0
- package/test/scenarios/reroute-second.json +16 -0
- package/test/scenarios/starvation-final.json +10 -0
- package/test/scenarios/starvation-promote.json +11 -0
- package/test/scenarios/wait-tools-continue.json +10 -0
- package/test/scenarios/wait-tools-xml.json +22 -0
- package/test/scenarios/wait-tools.json +15 -0
- package/test/session-format.test.ts +234 -0
- package/test/state.test.ts +335 -0
- package/test/stream-v02.test.ts +264 -0
- package/test/v031-display.test.ts +333 -0
- package/ui/client.js +576 -0
- package/ui/index.js +282 -0
- package/ui/package.json +26 -0
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
// eval/tools/runner.cjs ���� ���� 08 ���� runner���������������ִ�� + ȡ֤ + ָ�����
|
|
2
|
+
//
|
|
3
|
+
// �÷���
|
|
4
|
+
// node eval/tools/runner.cjs --arm baseline|stream [--stamp <2026-11-14>] [--tasks W1,R1] [--timeout 600]
|
|
5
|
+
// --stamp 2026-11-14a ��ȱʡ = �������ڣ�ͬ stamp ���ܸ���ͬ����Ŀ¼������ͬ�ո��ܣ�
|
|
6
|
+
// --key <DEEPSEEK_API_KEY>��ȱʡ�� process.env������ͬһ key = ���Ʊ������� 4��
|
|
7
|
+
//
|
|
8
|
+
// ÿ������� eval/runs/<stamp>/<arm>/<TASK>/��
|
|
9
|
+
// ws/ ����������seed ��ʵ�ܣ�������ԭ�� + manifest.json��
|
|
10
|
+
// probe/ eval-probe ���̣�probe.jsonl�������� options ԭ�� + �� chunk usage ԭ�ģ�
|
|
11
|
+
// driver-*.json ���ձۣ�06 ������ turn-end snapshot��STREAM_LOOP_EVAL_DUMP��
|
|
12
|
+
// session-log/session.v4.jsonl.zstd ԭ���浵���ط� oracle�����ߣ�zstd ԭ��������
|
|
13
|
+
// metrics.json ��Ԫ�� + prefix ���� + checks + �浵·����gen-results �����룩
|
|
14
|
+
|
|
15
|
+
const fs = require('node:fs');
|
|
16
|
+
const path = require('node:path');
|
|
17
|
+
const crypto = require('node:crypto');
|
|
18
|
+
const { spawnSync } = require('node:child_process');
|
|
19
|
+
const { newestSessionDir, parseSessionLog, summarizeSession, workspaceManifest } = require('./session-log.cjs');
|
|
20
|
+
|
|
21
|
+
const PKG_ROOT = path.resolve(__dirname, '..', '..');
|
|
22
|
+
const PATCH_BASELINE = path.join(PKG_ROOT, 'patches', 'eval-baseline.yml');
|
|
23
|
+
const PATCH_STREAM = path.join(PKG_ROOT, 'patches', 'eval-stream.yml');
|
|
24
|
+
const TASKS_FILE = path.join(PKG_ROOT, 'eval', 'tasks', 'tasks.json');
|
|
25
|
+
const FIXTURES = path.join(PKG_ROOT, 'eval', 'tasks', 'fixtures');
|
|
26
|
+
|
|
27
|
+
function arg(name, def) {
|
|
28
|
+
const i = process.argv.indexOf(`--${name}`);
|
|
29
|
+
return i >= 0 && i + 1 < process.argv.length ? process.argv[i + 1] : def;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** ̽�������Ƿ�Ϊ������·��METRICS.md M1/M2 ���ʶ�������� E5�� */
|
|
33
|
+
function isTitleRequest(optionsText) {
|
|
34
|
+
try {
|
|
35
|
+
const o = JSON.parse(optionsText);
|
|
36
|
+
const msgs = o.messages || [];
|
|
37
|
+
if (o.maxTokens === 64 && msgs.length <= 2) return true;
|
|
38
|
+
const text = JSON.stringify(msgs);
|
|
39
|
+
return text.includes('Generate the session title') || text.includes('Create a concise title');
|
|
40
|
+
} catch { return false; }
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** ��������ǰ���ֽڼ���M4������ probe ��� options ���л��ı�Ϊ */
|
|
44
|
+
function prefixSequence(requests) {
|
|
45
|
+
const out = [];
|
|
46
|
+
let prev = null;
|
|
47
|
+
for (const r of requests) {
|
|
48
|
+
const messages = r.messages; // probe request options ԭ�ģ�JSON �ַ�����
|
|
49
|
+
const hash = crypto.createHash('sha256').update(messages).digest('hex').slice(0, 16);
|
|
50
|
+
let common = 0;
|
|
51
|
+
if (prev !== null) {
|
|
52
|
+
const n = Math.min(prev.length, messages.length);
|
|
53
|
+
while (common < n && prev[common] === messages[common]) common += 1;
|
|
54
|
+
}
|
|
55
|
+
out.push({ seq: r.seq, fullHash: hash, commonPrefixBytesWithPrev: prev === null ? null : common, identicalToPrev: prev !== null && prev === messages });
|
|
56
|
+
prev = messages;
|
|
57
|
+
}
|
|
58
|
+
return out;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function sleepSync(ms) {
|
|
62
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function runTask(task, arm, runDir, apiKey, timeoutSec) {
|
|
66
|
+
const taskDir = path.join(runDir, arm, task.id);
|
|
67
|
+
const wsDir = path.join(taskDir, 'ws');
|
|
68
|
+
const probeDir = path.join(taskDir, 'probe');
|
|
69
|
+
fs.rmSync(taskDir, { recursive: true, force: true });
|
|
70
|
+
fs.mkdirSync(wsDir, { recursive: true });
|
|
71
|
+
fs.mkdirSync(probeDir, { recursive: true });
|
|
72
|
+
|
|
73
|
+
// seed���������ֽ�ͬһ�� fixtures��
|
|
74
|
+
for (const s of task.seed || []) {
|
|
75
|
+
const dst = path.join(wsDir, s.path);
|
|
76
|
+
fs.mkdirSync(path.dirname(dst), { recursive: true });
|
|
77
|
+
fs.copyFileSync(path.join(FIXTURES, s.fixture), dst);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const patch = arm === 'baseline' ? PATCH_BASELINE : PATCH_STREAM;
|
|
81
|
+
const env = { ...process.env, DSH_EVAL_PROBE_DIR: probeDir };
|
|
82
|
+
if (arm === 'stream') env.STREAM_LOOP_EVAL_DUMP = path.join(taskDir, 'driver-');
|
|
83
|
+
if (apiKey) env.DEEPSEEK_API_KEY = apiKey;
|
|
84
|
+
else delete env.DEEPSEEK_API_KEY;
|
|
85
|
+
|
|
86
|
+
const started = new Date().toISOString();
|
|
87
|
+
const res = spawnSync('dsh', ['--profile', 'headless', '--patch', patch, 'headless', task.prompt],
|
|
88
|
+
{ cwd: wsDir, env, timeout: timeoutSec * 1000, shell: true, encoding: 'utf8', windowsHide: true });
|
|
89
|
+
const finished = new Date().toISOString();
|
|
90
|
+
const timedOut = res.error && res.error.code === 'ETIMEDOUT';
|
|
91
|
+
|
|
92
|
+
// ---- ȡ֤ ----
|
|
93
|
+
// session log �ڽ����˳����������ӳ٣�ʵ�⣩����ѯ��� 30s ֱ������
|
|
94
|
+
let session = newestSessionDir(wsDir);
|
|
95
|
+
for (let waited = 0; session === undefined && waited < 30000; waited += 2000) {
|
|
96
|
+
sleepSync(2000);
|
|
97
|
+
session = newestSessionDir(wsDir);
|
|
98
|
+
}
|
|
99
|
+
let archived = { archived: false };
|
|
100
|
+
if (session !== undefined) {
|
|
101
|
+
const archDir = path.join(taskDir, 'session-log');
|
|
102
|
+
fs.mkdirSync(archDir, { recursive: true });
|
|
103
|
+
fs.copyFileSync(session.logPath, path.join(archDir, 'session.v4.jsonl.zstd')); // ԭ���ֽ�
|
|
104
|
+
archived = { archived: true, sessionDir: session.sessionDir, copiedBytes: fs.statSync(path.join(archDir, 'session.v4.jsonl.zstd')).size };
|
|
105
|
+
}
|
|
106
|
+
const manifest = workspaceManifest(wsDir);
|
|
107
|
+
fs.writeFileSync(path.join(wsDir, 'manifest.json'), JSON.stringify(manifest, null, 2), 'utf8');
|
|
108
|
+
|
|
109
|
+
// ---- session log ��ʵ ----
|
|
110
|
+
let sessionFacts = { error: 'session log δ�ҵ���archived=false ʱ�����ɸ��㣬��ʵ��ע��' };
|
|
111
|
+
if (session !== undefined) {
|
|
112
|
+
sessionFacts = summarizeSession(parseSessionLog(session.logPath));
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// ---- probe ��ʵ��M1 ���ھ� / M2 v1.1 ���ھ� / M4��----
|
|
116
|
+
const probeFile = path.join(probeDir, 'probe.jsonl');
|
|
117
|
+
const probeLines = fs.existsSync(probeFile)
|
|
118
|
+
? fs.readFileSync(probeFile, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return { kind: 'PARSE_ERROR' }; } })
|
|
119
|
+
: [];
|
|
120
|
+
const probeRequestsRaw = probeLines.filter((l) => l.kind === 'request');
|
|
121
|
+
const titleExcluded = probeRequestsRaw.filter((r) => isTitleRequest(r.options)).length;
|
|
122
|
+
const probeRequests = probeRequestsRaw
|
|
123
|
+
.filter((r) => !isTitleRequest(r.options))
|
|
124
|
+
.map((r) => ({ seq: r.seq, at: r.at, messages: r.options, usage: [] }));
|
|
125
|
+
const usageBySeq = new Map(probeRequests.map((r) => [r.seq, r]));
|
|
126
|
+
for (const l of probeLines) {
|
|
127
|
+
if (l.kind === 'chunk') {
|
|
128
|
+
try {
|
|
129
|
+
const c = JSON.parse(l.chunk);
|
|
130
|
+
if (c.type === 'usage' && usageBySeq.has(l.seq)) usageBySeq.get(l.seq).usage.push(c.usage);
|
|
131
|
+
} catch { /* ԭ�ı����� probe.jsonl������ʧ����ʵ��¼ PARSE_ERROR */ }
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// ---- M1 ���ܣ�E3 �ھ���input = uncached + cacheRead + cacheWrite��----
|
|
136
|
+
const tokens = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
|
|
137
|
+
let cachedFieldsSeen = false;
|
|
138
|
+
const usageOriginal = [];
|
|
139
|
+
for (const r of probeRequests) {
|
|
140
|
+
for (const u of r.usage) {
|
|
141
|
+
usageOriginal.push({ seq: r.seq, usage: u });
|
|
142
|
+
tokens.input += (u.inputTokens || 0) + (u.cacheReadTokens || 0) + (u.cacheWriteTokens || 0);
|
|
143
|
+
tokens.output += u.outputTokens || 0;
|
|
144
|
+
tokens.cacheRead += u.cacheReadTokens || 0;
|
|
145
|
+
tokens.cacheWrite += u.cacheWriteTokens || 0;
|
|
146
|
+
if (u.cacheReadTokens !== undefined || u.cacheWriteTokens !== undefined) cachedFieldsSeen = true;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// ---- M3�������� snapshot�����ձۣ�----
|
|
151
|
+
let driverDump = null;
|
|
152
|
+
if (arm === 'stream') {
|
|
153
|
+
const dumps = fs.readdirSync(taskDir).filter((n) => n.startsWith('driver-') && n.endsWith('.json'));
|
|
154
|
+
if (dumps.length > 0) driverDump = JSON.parse(fs.readFileSync(path.join(taskDir, dumps[dumps.length - 1]), 'utf8'));
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// ---- checks ----
|
|
158
|
+
const checks = (task.checks || []).map((c) => {
|
|
159
|
+
if (c.kind === 'exists') {
|
|
160
|
+
return { ...c, pass: fs.existsSync(path.join(wsDir, c.path)) };
|
|
161
|
+
}
|
|
162
|
+
if (c.kind === 'contains') {
|
|
163
|
+
const p = path.join(wsDir, c.path);
|
|
164
|
+
const pass = fs.existsSync(p) && fs.readFileSync(p, 'utf8').includes(c.text);
|
|
165
|
+
return { ...c, pass };
|
|
166
|
+
}
|
|
167
|
+
if (c.kind === 'final-contains') {
|
|
168
|
+
const finalText = sessionFacts.finalMessages ? sessionFacts.finalMessages.map((m) => m.text).join('\n') : '';
|
|
169
|
+
return { ...c, pass: finalText.includes(c.text), evidence: finalText.slice(0, 400) };
|
|
170
|
+
}
|
|
171
|
+
return { ...c, pass: false, note: 'unknown check kind' };
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
const metrics = {
|
|
175
|
+
task: task.id, arm, type: task.type,
|
|
176
|
+
started, finished, exitCode: res.status, timedOut: timedOut === true, signal: res.signal || null,
|
|
177
|
+
stdoutTail: (res.stdout || '').slice(-800), stderrTail: (res.stderr || '').slice(-1500),
|
|
178
|
+
M2_callCount: { main: sessionFacts.requestHeaderCount, probe: probeRequests.length, probeRaw: probeRequestsRaw.length, titleExcluded, note: 'v1.1 主口径=probe 计数;main(session log request/header) 仅基线臂可得(对账)' },
|
|
179
|
+
M1_tokens: { ...tokens, unitTaskToken: tokens.input + tokens.output, cachedFieldsSeen, usageOriginal, sourceNote: 'E3 口径:input=uncached+cacheRead+cacheWrite;usage 原文逐条留存' },
|
|
180
|
+
M3_injectedRatio: driverDump
|
|
181
|
+
? { injected: driverDump.counters.injected, produced: driverDump.counters.produced, injectedRatio: driverDump.counters.injectedRatio, source: 'driver snapshot(权威口径)' }
|
|
182
|
+
: (arm === 'baseline' ? { note: 'n/a(基线臂无该口径)' } : { note: 'driver dump 缺失' }),
|
|
183
|
+
M4_prefix: {
|
|
184
|
+
sequence: prefixSequence(probeRequests),
|
|
185
|
+
monotonicNote: '两臂 fullHash 应各自恒定(同追加式历史);commonPrefixBytesWithPrev 反映前缀单调性',
|
|
186
|
+
cachedFieldsEvidence: cachedFieldsSeen ? 'probe usage 原文含 cache 字段' : 'provider usage 未见 cache 字段(如实记录于 usageOriginal)',
|
|
187
|
+
},
|
|
188
|
+
modelConfig: sessionFacts.providers || [],
|
|
189
|
+
checks, checksPass: checks.every((c) => c.pass),
|
|
190
|
+
sessionArchived: archived,
|
|
191
|
+
sessionFactsNote: sessionFacts.error ? sessionFacts.error : undefined,
|
|
192
|
+
replay: { pending: true, note: '由 replay-check.cjs 生成' },
|
|
193
|
+
};
|
|
194
|
+
fs.writeFileSync(path.join(taskDir, 'metrics.json'), JSON.stringify(metrics, null, 2), 'utf8');
|
|
195
|
+
console.log(`[${arm}/${task.id}] exit=${res.status} calls=${metrics.M2_callCount.probe}(recon:${metrics.M2_callCount.main}) tokens=${metrics.M1_tokens.unitTaskToken} checks=${metrics.checksPass ? 'PASS' : 'FAIL'} archived=${archived.archived}`);
|
|
196
|
+
return metrics;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function main() {
|
|
200
|
+
const arm = arg('arm', '');
|
|
201
|
+
if (arm !== 'baseline' && arm !== 'stream') {
|
|
202
|
+
console.error('--arm ������ baseline �� stream');
|
|
203
|
+
process.exit(2);
|
|
204
|
+
}
|
|
205
|
+
const all = JSON.parse(fs.readFileSync(TASKS_FILE, 'utf8'));
|
|
206
|
+
const picked = arg('tasks', 'all') === 'all' ? all.tasks : all.tasks.filter((t) => arg('tasks', '').split(',').includes(t.id));
|
|
207
|
+
const stamp = arg('stamp', new Date().toISOString().slice(0, 10));
|
|
208
|
+
const runDir = path.join(PKG_ROOT, 'eval', 'runs', stamp);
|
|
209
|
+
fs.mkdirSync(runDir, { recursive: true });
|
|
210
|
+
const timeoutSec = Number(arg('timeout', '600'));
|
|
211
|
+
const apiKey = arg('key', '') || process.env.DEEPSEEK_API_KEY || '';
|
|
212
|
+
|
|
213
|
+
console.log(`runner: arm=${arm} tasks=${picked.map((t) => t.id).join(',')} stamp=${stamp} key=${apiKey ? 'SET(****' + apiKey.slice(-4) + ')' : 'NOT SET'}`);
|
|
214
|
+
const results = picked.map((t) => runTask(t, arm, runDir, apiKey, timeoutSec));
|
|
215
|
+
fs.writeFileSync(path.join(runDir, `${arm}-summary.json`), JSON.stringify({ arm, stamp, startedAt: new Date().toISOString(), results }, null, 2), 'utf8');
|
|
216
|
+
console.log(`summary -> ${path.join(runDir, arm + '-summary.json')}`);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
if (require.main === module) main();
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// eval/tools/session-log.cjs —— session log 解析与工作区取证(分派 08 工具链共用库)
|
|
2
|
+
// 事实锚点(METRICS.md E1/E2):
|
|
3
|
+
// - session log: %USERPROFILE%\.dsh\sessions\<slug>\session-*\session.v4.jsonl.zstd(zstd JSONL 原样保留)
|
|
4
|
+
// - request/header 不含 messages;usage chunk 在 assistant/attempt.data.stream 里
|
|
5
|
+
const fs = require('node:fs');
|
|
6
|
+
const path = require('node:path');
|
|
7
|
+
const zlib = require('node:zlib');
|
|
8
|
+
const crypto = require('node:crypto');
|
|
9
|
+
|
|
10
|
+
/** cwd → session 工作区 slug(实测对照 %USERPROFILE%\.dsh\sessions 目录名:'--'+路径净化+'--';
|
|
11
|
+
* 实测规则:冒号丢弃(C:\Users → C-Users,双字符不产生双横线),\/空白 → '-') */
|
|
12
|
+
function workspaceSlug(cwd) {
|
|
13
|
+
return '--' + cwd.replace(/:/g, '').replace(/[/\\\s]/g, '-') + '--';
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** 找 cwd 下最新的 session 目录(按 session.v4.jsonl.zstd mtime) */
|
|
17
|
+
function newestSessionDir(cwd, userProfile = process.env.USERPROFILE || '') {
|
|
18
|
+
const root = path.join(userProfile, '.dsh', 'sessions', workspaceSlug(cwd));
|
|
19
|
+
if (!fs.existsSync(root)) return undefined;
|
|
20
|
+
let best;
|
|
21
|
+
let bestTime = -1;
|
|
22
|
+
for (const name of fs.readdirSync(root)) {
|
|
23
|
+
const p = path.join(root, name, 'session.v4.jsonl.zstd');
|
|
24
|
+
if (!fs.existsSync(p)) continue;
|
|
25
|
+
const t = fs.statSync(p).mtimeMs;
|
|
26
|
+
if (t > bestTime) { bestTime = t; best = { sessionDir: path.join(root, name), logPath: p, mtime: t }; }
|
|
27
|
+
}
|
|
28
|
+
return best;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** 解压解析 session log → 记录数组(zstd 原样文件解压,只读) */
|
|
32
|
+
function parseSessionLog(logPath) {
|
|
33
|
+
const buf = zlib.zstdDecompressSync(fs.readFileSync(logPath));
|
|
34
|
+
const records = [];
|
|
35
|
+
for (const line of buf.toString('utf8').split('\n')) {
|
|
36
|
+
if (line.trim() === '') continue;
|
|
37
|
+
try { records.push(JSON.parse(line)); } catch { records.push({ type: 'PARSE_ERROR', raw: line.slice(0, 500) }); }
|
|
38
|
+
}
|
|
39
|
+
return records;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** 文本抽取:content blocks → 纯文本 */
|
|
43
|
+
function contentText(content) {
|
|
44
|
+
if (content === undefined || content === null) return '';
|
|
45
|
+
if (typeof content === 'string') return content;
|
|
46
|
+
if (Array.isArray(content)) {
|
|
47
|
+
return content.map((b) => (b && typeof b === 'object' && typeof b.text === 'string' ? b.text : '')).filter(Boolean).join('\n');
|
|
48
|
+
}
|
|
49
|
+
return String(content);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** 从 session log 记录提取评测四元组所需事实(METRICS.md M1/M2/任务成败) */
|
|
53
|
+
function summarizeSession(records) {
|
|
54
|
+
const out = {
|
|
55
|
+
requestHeaderCount: 0,
|
|
56
|
+
providers: [], // request/header 的 provider/model(控制变量取证)
|
|
57
|
+
usageChunks: [], // assistant/attempt 里 type=usage 的原文(M1 对账 + M4 cached 字段取证)
|
|
58
|
+
finishReasons: [],
|
|
59
|
+
finalMessages: [], // assistant/message 全文(final-contains 检查输入)
|
|
60
|
+
turnEnds: [],
|
|
61
|
+
titleRequests: [], // session/title-llm-request(M1 剔除项取证)
|
|
62
|
+
recordTypes: {},
|
|
63
|
+
};
|
|
64
|
+
for (const r of records) {
|
|
65
|
+
out.recordTypes[r.type] = (out.recordTypes[r.type] || 0) + 1;
|
|
66
|
+
if (r.type === 'request/header') {
|
|
67
|
+
out.requestHeaderCount += 1;
|
|
68
|
+
const cfg = r.data && r.data.header && r.data.header.config ? r.data.header.config : {};
|
|
69
|
+
out.providers.push({ provider: cfg.provider, model: cfg.model, at: r.time });
|
|
70
|
+
} else if (r.type === 'assistant/attempt') {
|
|
71
|
+
for (const s of (r.data && r.data.stream) || []) {
|
|
72
|
+
if (s.chunk && s.chunk.type === 'usage') out.usageChunks.push({ at: s.time, usage: s.chunk.usage });
|
|
73
|
+
if (s.chunk && s.chunk.type === 'finish') out.finishReasons.push({ at: s.time, reason: s.chunk.reason });
|
|
74
|
+
}
|
|
75
|
+
} else if (r.type === 'assistant/message') {
|
|
76
|
+
out.finalMessages.push({ turn: r.data && r.data.turn, text: contentText(r.data && r.data.message && r.data.message.content) });
|
|
77
|
+
} else if (r.type === 'turn/end') {
|
|
78
|
+
out.turnEnds.push({ turn: r.data && r.data.turn, reason: r.data && r.data.reason });
|
|
79
|
+
} else if (r.type === 'session/title-llm-request') {
|
|
80
|
+
out.titleRequests.push({ at: r.time, route: r.data && r.data.route });
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return out;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** 工作区树哈希:相对路径排序 + 逐文件 sha256 → 单一 manifest(M5 的"实跑后哈希") */
|
|
87
|
+
function workspaceManifest(wsDir) {
|
|
88
|
+
const files = [];
|
|
89
|
+
(function walk(dir) {
|
|
90
|
+
for (const name of fs.readdirSync(dir)) {
|
|
91
|
+
const p = path.join(dir, name);
|
|
92
|
+
const st = fs.statSync(p);
|
|
93
|
+
if (st.isDirectory()) walk(p);
|
|
94
|
+
else files.push({ rel: path.relative(wsDir, p).replace(/\\/g, '/'), size: st.size });
|
|
95
|
+
}
|
|
96
|
+
})(wsDir);
|
|
97
|
+
files.sort((a, b) => (a.rel < b.rel ? -1 : 1));
|
|
98
|
+
const entries = files.map((f) => {
|
|
99
|
+
const hash = crypto.createHash('sha256').update(fs.readFileSync(path.join(wsDir, f.rel))).digest('hex');
|
|
100
|
+
return { ...f, sha256: hash };
|
|
101
|
+
});
|
|
102
|
+
const root = crypto.createHash('sha256').update(entries.map((e) => `${e.rel}:${e.sha256}`).join('\n')).digest('hex');
|
|
103
|
+
return { root, files: entries };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
module.exports = { workspaceSlug, newestSessionDir, parseSessionLog, summarizeSession, contentText, workspaceManifest };
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
// eval/tools/summarize.cjs —— M1 四分量计量汇总(输入/缓存读/缓存写/输出;README 数字由此出)
|
|
2
|
+
//
|
|
3
|
+
// 用法:
|
|
4
|
+
// node eval/tools/summarize.cjs --stamp <2026-11-15> [--arm baseline|stream|all] [--md]
|
|
5
|
+
//
|
|
6
|
+
// 输出:
|
|
7
|
+
// eval/runs/<stamp>/summary.json —— 逐任务 + 汇总的四分量原始数(可复算的权威形态)
|
|
8
|
+
// --md 时同时打印 Markdown 表(贴进 eval/RESULTS.md 用)
|
|
9
|
+
//
|
|
10
|
+
// 口径(eval/METRICS.md M1):
|
|
11
|
+
// 单位任务 token = Σ 请求 [inputTokens + cacheReadTokens + cacheWriteTokens + outputTokens]
|
|
12
|
+
// · 逐请求取 probe.jsonl 的 usage chunk(eval-probe 抓原文,留存可复算);
|
|
13
|
+
// · 会话标题旁路请求剔除(识别特征见 isTitleRequest,两臂同剔);
|
|
14
|
+
// · 缓存分量缺失标 "不可得"(按 0 计会系统性低估,禁止);
|
|
15
|
+
// · 二分量(输入+输出)只作参考并显式标注 "非全量口径"。
|
|
16
|
+
|
|
17
|
+
const fs = require('node:fs');
|
|
18
|
+
const path = require('node:path');
|
|
19
|
+
|
|
20
|
+
const PKG_ROOT = path.resolve(__dirname, '..', '..');
|
|
21
|
+
|
|
22
|
+
function arg(name, def) {
|
|
23
|
+
const i = process.argv.indexOf(`--${name}`);
|
|
24
|
+
return i >= 0 && i + 1 < process.argv.length ? process.argv[i + 1] : def;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** 标题旁路请求识别(METRICS.md E5;两臂同剔) */
|
|
28
|
+
function isTitleRequest(optionsText) {
|
|
29
|
+
try {
|
|
30
|
+
const o = JSON.parse(optionsText);
|
|
31
|
+
const msgs = o.messages || [];
|
|
32
|
+
if (o.maxTokens === 64 && msgs.length <= 2) return true;
|
|
33
|
+
const text = JSON.stringify(msgs);
|
|
34
|
+
return text.includes('Generate the session title') || text.includes('Create a concise title');
|
|
35
|
+
} catch {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** 单任务四分量汇总:逐请求 usage 原文求和;缺失分量如实标 null("不可得")。 */
|
|
41
|
+
function summarizeTask(probePath) {
|
|
42
|
+
if (!fs.existsSync(probePath)) return null;
|
|
43
|
+
const components = {
|
|
44
|
+
inputTokens: { sum: 0, seen: false },
|
|
45
|
+
cacheReadTokens: { sum: 0, seen: false },
|
|
46
|
+
cacheWriteTokens: { sum: 0, seen: false },
|
|
47
|
+
outputTokens: { sum: 0, seen: false },
|
|
48
|
+
};
|
|
49
|
+
let requests = 0;
|
|
50
|
+
let excludedTitle = 0;
|
|
51
|
+
const currentSeqTitle = new Set();
|
|
52
|
+
const lines = fs.readFileSync(probePath, 'utf8').split('\n');
|
|
53
|
+
// 先扫一遍 request 行标出标题请求的 seq(usage chunk 与 request 同 seq)
|
|
54
|
+
for (const line of lines) {
|
|
55
|
+
if (line === '') continue;
|
|
56
|
+
let rec;
|
|
57
|
+
try { rec = JSON.parse(line); } catch { continue; }
|
|
58
|
+
if (rec.kind === 'request' && isTitleRequest(rec.options)) currentSeqTitle.add(rec.seq);
|
|
59
|
+
}
|
|
60
|
+
for (const line of lines) {
|
|
61
|
+
if (line === '') continue;
|
|
62
|
+
let rec;
|
|
63
|
+
try { rec = JSON.parse(line); } catch { continue; }
|
|
64
|
+
if (rec.kind === 'request' && !currentSeqTitle.has(rec.seq)) requests += 1;
|
|
65
|
+
if (rec.kind === 'request' && currentSeqTitle.has(rec.seq)) excludedTitle += 1;
|
|
66
|
+
if (rec.kind !== 'chunk' || currentSeqTitle.has(rec.seq)) continue;
|
|
67
|
+
let chunk;
|
|
68
|
+
try { chunk = JSON.parse(rec.chunk); } catch { continue; }
|
|
69
|
+
if (chunk.type !== 'usage' || chunk.usage === undefined) continue;
|
|
70
|
+
for (const key of Object.keys(components)) {
|
|
71
|
+
const v = chunk.usage[key];
|
|
72
|
+
if (typeof v === 'number') {
|
|
73
|
+
components[key].sum += v;
|
|
74
|
+
components[key].seen = true;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
const pick = (key) => (components[key].seen ? components[key].sum : null);
|
|
79
|
+
const input = pick('inputTokens');
|
|
80
|
+
const cacheRead = pick('cacheReadTokens');
|
|
81
|
+
const cacheWrite = pick('cacheWriteTokens');
|
|
82
|
+
const output = pick('outputTokens');
|
|
83
|
+
const unavailable = [cacheRead, cacheWrite].filter((v) => v === null).length;
|
|
84
|
+
const four = input === null || output === null
|
|
85
|
+
? null
|
|
86
|
+
: input + (cacheRead ?? 0) + (cacheWrite === null ? 0 : cacheWrite) + output;
|
|
87
|
+
return {
|
|
88
|
+
requests,
|
|
89
|
+
excludedTitleRequests: excludedTitle,
|
|
90
|
+
inputTokens: input,
|
|
91
|
+
cacheReadTokens: cacheRead,
|
|
92
|
+
cacheWriteTokens: cacheWrite,
|
|
93
|
+
outputTokens: output,
|
|
94
|
+
fourComponentTotal: four,
|
|
95
|
+
twoComponentTotal: input === null || output === null ? null : input + output,
|
|
96
|
+
cacheComponentsUnavailable: unavailable > 0,
|
|
97
|
+
note: unavailable > 0 ? '缓存分量不可得(未按 0 计)' : undefined,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function main() {
|
|
102
|
+
const stamp = arg('stamp');
|
|
103
|
+
if (stamp === undefined) {
|
|
104
|
+
console.error('用法:node eval/tools/summarize.cjs --stamp <2026-11-15> [--arm all] [--md]');
|
|
105
|
+
process.exit(2);
|
|
106
|
+
}
|
|
107
|
+
const runDir = path.join(PKG_ROOT, 'eval', 'runs', stamp);
|
|
108
|
+
if (!fs.existsSync(runDir)) {
|
|
109
|
+
console.error(`[summarize] 目录不存在:${runDir}`);
|
|
110
|
+
process.exit(2);
|
|
111
|
+
}
|
|
112
|
+
const armArg = arg('arm', 'all');
|
|
113
|
+
const arms = armArg === 'all' ? ['baseline', 'stream'] : [armArg];
|
|
114
|
+
const tasks = JSON.parse(fs.readFileSync(path.join(PKG_ROOT, 'eval', 'tasks', 'tasks.json'), 'utf8')).tasks;
|
|
115
|
+
|
|
116
|
+
const out = { stamp, arms: {} };
|
|
117
|
+
for (const arm of arms) {
|
|
118
|
+
const rows = {};
|
|
119
|
+
let reqTotal = 0;
|
|
120
|
+
let checksPass = 0;
|
|
121
|
+
let checksTotal = 0;
|
|
122
|
+
for (const task of tasks) {
|
|
123
|
+
const metricsPath = path.join(runDir, arm, task.id, 'metrics.json');
|
|
124
|
+
const probePath = path.join(runDir, arm, task.id, 'probe', 'probe.jsonl');
|
|
125
|
+
const row = summarizeTask(probePath);
|
|
126
|
+
if (row === null) {
|
|
127
|
+
rows[task.id] = { missing: true };
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
if (fs.existsSync(metricsPath)) {
|
|
131
|
+
const metrics = JSON.parse(fs.readFileSync(metricsPath, 'utf8'));
|
|
132
|
+
const checks = metrics.checks ?? [];
|
|
133
|
+
checksTotal += checks.length;
|
|
134
|
+
checksPass += checks.filter((c) => c.pass === true).length;
|
|
135
|
+
}
|
|
136
|
+
rows[task.id] = row;
|
|
137
|
+
reqTotal += row.requests;
|
|
138
|
+
}
|
|
139
|
+
const totals = {};
|
|
140
|
+
for (const key of ['inputTokens', 'cacheReadTokens', 'cacheWriteTokens', 'outputTokens']) {
|
|
141
|
+
const vals = Object.values(rows).filter((r) => !r.missing && r[key] !== null);
|
|
142
|
+
totals[key] = vals.length === Object.keys(rows).filter((r) => !r.missing).length
|
|
143
|
+
? vals.reduce((s, r) => s + r[key], 0)
|
|
144
|
+
: null; // 任一任务缺该分量 → 汇总也不可加总(不得补 0)
|
|
145
|
+
}
|
|
146
|
+
const t2 = Object.values(rows).filter((r) => !r.missing && r.twoComponentTotal !== null);
|
|
147
|
+
const t2Sum = t2.reduce((s, r) => s + r.twoComponentTotal, 0);
|
|
148
|
+
const full = totals.inputTokens !== null && totals.outputTokens !== null
|
|
149
|
+
&& totals.cacheReadTokens !== null && totals.cacheWriteTokens !== null;
|
|
150
|
+
out.arms[arm] = {
|
|
151
|
+
tasks: rows,
|
|
152
|
+
totals: { ...totals, fourComponentTotal: full ? totals.inputTokens + totals.cacheReadTokens + totals.cacheWriteTokens + totals.outputTokens : null },
|
|
153
|
+
requests: reqTotal,
|
|
154
|
+
checks: { pass: checksPass, total: checksTotal },
|
|
155
|
+
twoComponentReference: { sum: t2Sum, note: '非全量口径,只作参考(METRICS.md M1)' },
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const outPath = path.join(runDir, 'summary.json');
|
|
160
|
+
fs.writeFileSync(outPath, `${JSON.stringify(out, null, 2)}\n`, 'utf8');
|
|
161
|
+
console.log(`[summarize] 写出 ${outPath}`);
|
|
162
|
+
|
|
163
|
+
if (process.argv.includes('--md')) {
|
|
164
|
+
for (const arm of arms) {
|
|
165
|
+
const a = out.arms[arm];
|
|
166
|
+
console.log(`\n### ${arm}\n`);
|
|
167
|
+
console.log('| 任务 | 请求数 | 输入 | 缓存读 | 缓存写 | 输出 | 全量合计 |');
|
|
168
|
+
console.log('|---|---|---|---|---|---|---|');
|
|
169
|
+
for (const task of tasks) {
|
|
170
|
+
const r = a.tasks[task.id];
|
|
171
|
+
if (r === undefined || r.missing) {
|
|
172
|
+
console.log(`| ${task.id} | - | - | - | - | - | - |`);
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
const cell = (v) => (v === null ? '不可得' : String(v));
|
|
176
|
+
console.log(`| ${task.id} | ${r.requests} | ${cell(r.inputTokens)} | ${cell(r.cacheReadTokens)} | ${cell(r.cacheWriteTokens)} | ${cell(r.outputTokens)} | ${cell(r.fourComponentTotal)} |`);
|
|
177
|
+
}
|
|
178
|
+
const t = a.totals;
|
|
179
|
+
console.log(`| 合计 | ${a.requests} | ${t.inputTokens ?? '不可得'} | ${t.cacheReadTokens ?? '不可得'} | ${t.cacheWriteTokens ?? '不可得'} | ${t.outputTokens ?? '不可得'} | ${t.fourComponentTotal ?? '不可得(缺失分量未按 0 计)'} |`);
|
|
180
|
+
console.log(`\nchecks:${a.checks.pass}/${a.checks.total};二分量参考(非全量口径):${a.twoComponentReference.sum}`);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
main();
|