niceeval 0.11.2 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +1 -0
- package/README.md +4 -2
- package/README.zh.md +4 -2
- package/dist/agents/types.d.ts +2 -2
- package/dist/o11y/types.d.ts +3 -3
- package/dist/report/assets/colors.d.ts +1 -1
- package/dist/report/assets/colors.js +1 -1
- package/dist/report/components/attempt-detail/AttemptSource.js +1 -2
- package/dist/report/components/attempt-detail/compute.d.ts +4 -4
- package/dist/report/components/attempt-detail/compute.js +6 -6
- package/dist/report/components/attempt-detail/faces.d.ts +1 -1
- package/dist/report/components/attempt-detail/faces.js +3 -3
- package/dist/report/components/entity-lists/AttemptList.d.ts +3 -3
- package/dist/report/components/entity-lists/AttemptList.js +3 -3
- package/dist/report/components/entity-lists/ExperimentList.js +2 -2
- package/dist/report/components/entity-lists/compute.js +1 -1
- package/dist/report/components/entity-lists/faces.js +1 -1
- package/dist/report/components/entity-lists/index.d.ts +1 -1
- package/dist/report/components/entity-lists/index.js +1 -1
- package/dist/report/components/metric-views/chart-math.d.ts +2 -2
- package/dist/report/components/metric-views/chart-math.js +2 -2
- package/dist/report/components/metric-views/compute.d.ts +2 -3
- package/dist/report/components/metric-views/compute.js +3 -4
- package/dist/report/components/shared.d.ts +1 -1
- package/dist/report/components/site-components/compute.d.ts +5 -5
- package/dist/report/components/site-components/compute.js +6 -6
- package/dist/report/components/site-components/index.d.ts +7 -7
- package/dist/report/components/site-components/index.js +7 -7
- package/dist/report/components/site-components/scope-warnings.d.ts +1 -1
- package/dist/report/components/site-components/scope-warnings.js +3 -3
- package/dist/report/components/site-components/snapshot-diagnostics.js +1 -1
- package/dist/report/components/summaries/compute.d.ts +1 -1
- package/dist/report/components/summaries/compute.js +2 -2
- package/dist/report/components/summaries/faces.js +1 -1
- package/dist/report/components/summaries/index.d.ts +1 -1
- package/dist/report/components/summaries/index.js +1 -1
- package/dist/report/index.js +1 -1
- package/dist/report/model/aggregate.d.ts +2 -2
- package/dist/report/model/aggregate.js +2 -2
- package/dist/report/model/format.d.ts +2 -2
- package/dist/report/model/format.js +2 -2
- package/dist/report/model/metrics.js +3 -3
- package/dist/report/model/types.d.ts +11 -12
- package/dist/results/locator.js +1 -1
- package/dist/results/select.d.ts +4 -4
- package/dist/results/select.js +5 -5
- package/dist/results/types.d.ts +8 -8
- package/dist/runner/types.d.ts +14 -14
- package/dist/sandbox/resolve.d.ts +1 -1
- package/dist/sandbox/types.d.ts +1 -1
- package/docs-site/zh/reference/builtin-agents.mdx +21 -0
- package/docs-site/zh/reference/define-agent.mdx +1 -1
- package/docs-site/zh/reference/report-components.mdx +156 -74
- package/docs-site/zh/tutorials/custom-reports.mdx +112 -22
- package/docs-site/zh/tutorials/sandbox-agent.mdx +3 -4
- package/docs-site/zh/tutorials/sandbox-providers.mdx +15 -15
- package/docs-site/zh/tutorials/theming.mdx +169 -0
- package/package.json +2 -1
- package/src/agents/ai-sdk.test.ts +1 -1
- package/src/agents/bub-install-spec.ts +26 -8
- package/src/agents/bub.ts +40 -15
- package/src/agents/builtin.ts +11 -1
- package/src/agents/coding-cli-versions.ts +67 -0
- package/src/agents/hermes.ts +235 -0
- package/src/agents/index.ts +4 -0
- package/src/agents/langgraph.test.ts +1 -1
- package/src/agents/manifest.ts +1 -1
- package/src/agents/openai-compat.test.ts +1 -1
- package/src/agents/openclaw.ts +103 -24
- package/src/agents/opencode.ts +183 -0
- package/src/agents/sdk-streams.test.ts +1 -1
- package/src/agents/shared.ts +13 -2
- package/src/agents/types.ts +2 -2
- package/src/context/session.test.ts +1 -1
- package/src/context/session.ts +1 -1
- package/src/o11y/cost.ts +1 -1
- package/src/o11y/parsers/bub.test.ts +1 -1
- package/src/o11y/parsers/bub.ts +1 -1
- package/src/o11y/parsers/codex.test.ts +1 -1
- package/src/o11y/parsers/codex.ts +1 -1
- package/src/o11y/parsers/hermes.ts +198 -0
- package/src/o11y/parsers/openclaw.ts +25 -3
- package/src/o11y/parsers/opencode.ts +295 -0
- package/src/o11y/types.ts +3 -3
- package/src/report/assets/colors.ts +1 -1
- package/src/report/assets/styles.css +1 -1
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptConversation.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptDiff.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptError.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptSource.tsx +2 -3
- package/src/report/components/attempt-detail/AttemptSummary.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptTrace.tsx +1 -1
- package/src/report/components/attempt-detail/UsageTable.tsx +1 -1
- package/src/report/components/attempt-detail/attempt-components.test.tsx +1 -1
- package/src/report/components/attempt-detail/compute.ts +6 -6
- package/src/report/components/attempt-detail/faces.ts +3 -3
- package/src/report/components/attempt-detail/index.tsx +1 -1
- package/src/report/components/entity-lists/AttemptList.tsx +3 -3
- package/src/report/components/entity-lists/ExperimentList.tsx +2 -2
- package/src/report/components/entity-lists/compute.ts +1 -1
- package/src/report/components/entity-lists/faces.ts +1 -1
- package/src/report/components/entity-lists/index.tsx +1 -1
- package/src/report/components/metric-views/DeltaTable.tsx +1 -2
- package/src/report/components/metric-views/StabilityMatrix.tsx +1 -1
- package/src/report/components/metric-views/chart-math.test.ts +1 -1
- package/src/report/components/metric-views/chart-math.ts +2 -2
- package/src/report/components/metric-views/compute.ts +3 -4
- package/src/report/components/shared.ts +1 -1
- package/src/report/components/site-components/CopyFixPrompt.tsx +1 -1
- package/src/report/components/site-components/HeroCard.tsx +1 -1
- package/src/report/components/site-components/PoweredBy.tsx +1 -1
- package/src/report/components/site-components/ScopeWarnings.tsx +1 -1
- package/src/report/components/site-components/SnapshotDiagnostics.tsx +1 -1
- package/src/report/components/site-components/TraceWaterfall.tsx +1 -1
- package/src/report/components/site-components/compute.ts +6 -6
- package/src/report/components/site-components/index.tsx +7 -7
- package/src/report/components/site-components/scope-warnings.ts +3 -3
- package/src/report/components/site-components/snapshot-diagnostics.ts +1 -1
- package/src/report/components/summaries/ScopeSummary.tsx +2 -2
- package/src/report/components/summaries/compute.ts +2 -2
- package/src/report/components/summaries/faces.ts +1 -1
- package/src/report/components/summaries/index.tsx +1 -1
- package/src/report/index.ts +1 -1
- package/src/report/model/aggregate.ts +2 -2
- package/src/report/model/format.ts +2 -2
- package/src/report/model/metrics.ts +3 -3
- package/src/report/model/types.ts +12 -13
- package/src/results/annotated-source.test.ts +1 -1
- package/src/results/attempt-evidence.test.ts +1 -1
- package/src/results/copy.ts +3 -3
- package/src/results/format.ts +3 -3
- package/src/results/host-equivalence.test.ts +6 -6
- package/src/results/index.ts +1 -1
- package/src/results/locator.test.ts +2 -2
- package/src/results/locator.ts +1 -1
- package/src/results/open.ts +2 -2
- package/src/results/publish.ts +1 -1
- package/src/results/results.test.ts +6 -323
- package/src/results/select.test.ts +438 -0
- package/src/results/select.ts +6 -6
- package/src/results/skipped-notice.ts +1 -1
- package/src/results/truncate.ts +2 -2
- package/src/results/types.ts +9 -9
- package/src/results/writer.ts +4 -4
- package/src/runner/attempt.test.ts +2 -2
- package/src/runner/attempt.ts +6 -6
- package/src/runner/eval-source.test.ts +1 -1
- package/src/runner/eval-source.ts +1 -1
- package/src/runner/reporters/artifacts.ts +1 -1
- package/src/runner/run.ts +2 -2
- package/src/runner/timing.ts +1 -1
- package/src/runner/types.ts +14 -14
- package/src/sandbox/docker-agent-image.ts +36 -0
- package/src/sandbox/e2b-agent-template.test.ts +6 -21
- package/src/sandbox/e2b-agent-template.ts +44 -11
- package/src/sandbox/index.ts +8 -0
- package/src/sandbox/official-baselines.test.ts +175 -0
- package/src/sandbox/resolve.ts +1 -1
- package/src/sandbox/types.ts +1 -1
- package/src/scoring/diff.ts +1 -1
- package/src/scoring/types.ts +1 -1
- package/src/shared/facts.ts +1 -1
- package/src/show/index.ts +1 -1
- package/src/show/render.ts +1 -1
- package/src/show/show.test.ts +3 -3
- package/src/view/data.test.ts +2 -2
- package/src/view/data.ts +1 -1
- package/src/view/view-report.test.ts +1 -1
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import { defineSandboxAgent } from "../define.ts";
|
|
2
|
+
import { requireEnv, getEnv } from "../util.ts";
|
|
3
|
+
import { shared } from "./shared.ts";
|
|
4
|
+
import {
|
|
5
|
+
appendProjectInstruction,
|
|
6
|
+
installSkills,
|
|
7
|
+
installedSkillNames,
|
|
8
|
+
skillDiscoveryInstruction,
|
|
9
|
+
} from "./skills.ts";
|
|
10
|
+
import { writeAgentSetupManifest } from "./manifest.ts";
|
|
11
|
+
import { mapGenericSpans } from "../o11y/otlp/canonical.ts";
|
|
12
|
+
import { completeCoverage } from "../scoring/coverage.ts";
|
|
13
|
+
import {
|
|
14
|
+
parseOpenCodeTranscript,
|
|
15
|
+
sessionIdFromOpenCodeTranscript,
|
|
16
|
+
extractOpenCodeJsonl,
|
|
17
|
+
} from "../o11y/parsers/opencode.ts";
|
|
18
|
+
import { DEFAULT_OPENCODE_CLI_VERSION } from "./coding-cli-versions.ts";
|
|
19
|
+
import type { Agent, AgentSetupManifest, SkillSpec, StreamEvent } from "../types.ts";
|
|
20
|
+
|
|
21
|
+
// OpenCode sandbox adapter。驱动:`opencode run --format json --auto`;
|
|
22
|
+
// 行为轨优先 stdout JSONL,不足时 `opencode export <sessionID>`。
|
|
23
|
+
// 契约见 docs/feature/adapters/sdk/opencode/README.md。
|
|
24
|
+
|
|
25
|
+
const SKILL_DIR = ".agents/skills";
|
|
26
|
+
/** 自定义 OpenAI 兼容网关在 opencode.json 里的 provider id。 */
|
|
27
|
+
const COMPAT_PROVIDER = "compat";
|
|
28
|
+
|
|
29
|
+
export interface OpenCodeConfig {
|
|
30
|
+
/** 模型 API key。省略时读 OPENCODE_API_KEY,再回落 ANTHROPIC_API_KEY。 */
|
|
31
|
+
apiKey?: string;
|
|
32
|
+
/** OpenAI 兼容端点。省略时读 OPENCODE_BASE_URL。 */
|
|
33
|
+
baseUrl?: string;
|
|
34
|
+
/** 钉 npm `opencode-ai` 版本;省略时用 NiceEval 默认 pin。 */
|
|
35
|
+
version?: string;
|
|
36
|
+
/** 装进沙箱的 Skill,落在 `.agents/skills/<name>/`。 */
|
|
37
|
+
skills?: SkillSpec[];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function resolveApiKey(config?: OpenCodeConfig): string {
|
|
41
|
+
return config?.apiKey ?? getEnv("OPENCODE_API_KEY") ?? requireEnv("ANTHROPIC_API_KEY");
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function resolveBaseUrl(config?: OpenCodeConfig): string | undefined {
|
|
45
|
+
return config?.baseUrl ?? getEnv("OPENCODE_BASE_URL");
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** experiment.model → `--model`。裸模型名在自定义网关下补成 `compat/<model>`。 */
|
|
49
|
+
function resolveModelFlag(model: string | undefined, hasCompatBase: boolean): string | undefined {
|
|
50
|
+
if (!model) return undefined;
|
|
51
|
+
if (model.includes("/")) return model;
|
|
52
|
+
return hasCompatBase ? `${COMPAT_PROVIDER}/${model}` : model;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* OpenCode 的内置 sandbox Agent 工厂。
|
|
57
|
+
*/
|
|
58
|
+
export function openCodeAgent(config?: OpenCodeConfig): Agent {
|
|
59
|
+
const version = config?.version ?? DEFAULT_OPENCODE_CLI_VERSION;
|
|
60
|
+
|
|
61
|
+
return defineSandboxAgent({
|
|
62
|
+
name: "opencode",
|
|
63
|
+
coverage: completeCoverage,
|
|
64
|
+
spanMapper: mapGenericSpans,
|
|
65
|
+
|
|
66
|
+
tracing: {
|
|
67
|
+
protocol: "http/protobuf",
|
|
68
|
+
env: (endpoint) => ({
|
|
69
|
+
OTEL_TRACES_EXPORTER: "otlp",
|
|
70
|
+
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: endpoint,
|
|
71
|
+
OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: "http/protobuf",
|
|
72
|
+
}),
|
|
73
|
+
},
|
|
74
|
+
|
|
75
|
+
async setup(sb) {
|
|
76
|
+
await sb.runShell(
|
|
77
|
+
`command -v opencode >/dev/null 2>&1 || npm install -g opencode-ai@${version}`,
|
|
78
|
+
);
|
|
79
|
+
|
|
80
|
+
const baseUrl = resolveBaseUrl(config);
|
|
81
|
+
const provider: Record<string, unknown> = {};
|
|
82
|
+
if (baseUrl) {
|
|
83
|
+
// OpenAI 兼容网关必须走 openai-compatible npm 包 + 显式 models 表,
|
|
84
|
+
// 否则 OpenCode 不会把自定义 baseURL 接到请求上。
|
|
85
|
+
provider[COMPAT_PROVIDER] = {
|
|
86
|
+
npm: "@ai-sdk/openai-compatible",
|
|
87
|
+
name: "OpenAI-compatible",
|
|
88
|
+
options: {
|
|
89
|
+
apiKey: "{env:OPENCODE_API_KEY}",
|
|
90
|
+
baseURL: baseUrl,
|
|
91
|
+
},
|
|
92
|
+
models: {
|
|
93
|
+
// 允许任意 experiment.model;具体 id 由 --model compat/<id> 选择。
|
|
94
|
+
"gpt-5.6-luna": { name: "gpt-5.6-luna" },
|
|
95
|
+
"gpt-5.4-mini": { name: "gpt-5.4-mini" },
|
|
96
|
+
"gpt-5.4": { name: "gpt-5.4" },
|
|
97
|
+
"gpt-4.1-mini": { name: "gpt-4.1-mini" },
|
|
98
|
+
"gpt-4o-mini": { name: "gpt-4o-mini" },
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
const opencodeConfig = {
|
|
103
|
+
$schema: "https://opencode.ai/config.json",
|
|
104
|
+
...(Object.keys(provider).length ? { provider } : {}),
|
|
105
|
+
permission: {
|
|
106
|
+
write: "allow",
|
|
107
|
+
edit: "allow",
|
|
108
|
+
bash: "allow",
|
|
109
|
+
read: "allow",
|
|
110
|
+
},
|
|
111
|
+
};
|
|
112
|
+
await shared.writeFile(sb, "opencode.json", JSON.stringify(opencodeConfig, null, 2));
|
|
113
|
+
|
|
114
|
+
const manifest: AgentSetupManifest = { skills: [] };
|
|
115
|
+
if (config?.skills?.length) {
|
|
116
|
+
manifest.skills = await installSkills(sb, config.skills, { dir: SKILL_DIR });
|
|
117
|
+
await appendProjectInstruction(
|
|
118
|
+
sb,
|
|
119
|
+
skillDiscoveryInstruction(SKILL_DIR, installedSkillNames(manifest.skills)),
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
if (manifest.skills.length) {
|
|
123
|
+
await writeAgentSetupManifest(sb, manifest);
|
|
124
|
+
}
|
|
125
|
+
},
|
|
126
|
+
|
|
127
|
+
async send(input, ctx) {
|
|
128
|
+
const sb = ctx.sandbox;
|
|
129
|
+
const baseUrl = resolveBaseUrl(config);
|
|
130
|
+
const args = ["run", input.text, "--format", "json", "--auto"];
|
|
131
|
+
const model = resolveModelFlag(ctx.model, Boolean(baseUrl));
|
|
132
|
+
if (model) args.push("--model", model);
|
|
133
|
+
if (ctx.session.id) args.push("--session", ctx.session.id);
|
|
134
|
+
|
|
135
|
+
const apiKey = resolveApiKey(config);
|
|
136
|
+
const env: Record<string, string> = {
|
|
137
|
+
OPENCODE_API_KEY: apiKey,
|
|
138
|
+
OPENAI_API_KEY: apiKey,
|
|
139
|
+
ANTHROPIC_API_KEY: apiKey,
|
|
140
|
+
...ctx.telemetry?.env,
|
|
141
|
+
};
|
|
142
|
+
if (baseUrl) {
|
|
143
|
+
env.OPENCODE_BASE_URL = baseUrl;
|
|
144
|
+
env.OPENAI_BASE_URL = baseUrl;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const res = await sb.runCommand("opencode", args, { env, stream: true });
|
|
148
|
+
let raw = extractOpenCodeJsonl(res.stdout) ?? extractOpenCodeJsonl(`${res.stdout}\n${res.stderr}`);
|
|
149
|
+
let sessionId = sessionIdFromOpenCodeTranscript(raw) ?? sessionIdFromOpenCodeTranscript(res.stdout);
|
|
150
|
+
if (sessionId) ctx.session.capture(sessionId);
|
|
151
|
+
|
|
152
|
+
let parsed = parseOpenCodeTranscript(raw);
|
|
153
|
+
// 仅当 stdout 既没有工具也没有助手文本时才 export 补读——纯对话轮
|
|
154
|
+
// (session/recall) 的 text 事件已在 JSONL 里,再 export 会冲掉已解析事件。
|
|
155
|
+
const hasActions = parsed.events.some((e) => e.type === "action.called");
|
|
156
|
+
const hasMessages = parsed.events.some((e) => e.type === "message");
|
|
157
|
+
if (!hasActions && !hasMessages && (sessionId ?? ctx.session.id)) {
|
|
158
|
+
const sid = sessionId ?? ctx.session.id!;
|
|
159
|
+
const exported = await sb.runCommand("opencode", ["export", sid], { env });
|
|
160
|
+
if (exported.exitCode === 0 && exported.stdout.trim()) {
|
|
161
|
+
raw = exported.stdout;
|
|
162
|
+
parsed = parseOpenCodeTranscript(raw);
|
|
163
|
+
sessionId = sessionIdFromOpenCodeTranscript(raw) ?? sessionId;
|
|
164
|
+
if (sessionId) ctx.session.capture(sessionId);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const events: StreamEvent[] = [...parsed.events];
|
|
169
|
+
const hasErrorEvent = events.some((e) => e.type === "error");
|
|
170
|
+
const failed = res.exitCode !== 0 || hasErrorEvent;
|
|
171
|
+
if (res.exitCode !== 0) {
|
|
172
|
+
events.push({ type: "error", message: shared.diagnoseFailure(res, parsed.events, raw) });
|
|
173
|
+
}
|
|
174
|
+
return {
|
|
175
|
+
events,
|
|
176
|
+
usage: parsed.usage,
|
|
177
|
+
status: failed ? "failed" : "completed",
|
|
178
|
+
};
|
|
179
|
+
},
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
export default openCodeAgent();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// cases: docs/engineering/testing/unit/
|
|
1
|
+
// cases: docs/engineering/testing/unit/record.md
|
|
2
2
|
// 「Usage、facts 与失败命令证据落盘」桶恒互斥归一:codex(OpenAI 口径,cached ⊂ input)扣减、
|
|
3
3
|
// Anthropic / pi(互斥口径)如实转发。fixture 数值刻意让「扣与不扣」结果可区分。
|
|
4
4
|
// bug: memory/estimatecost-openai-inclusive-cache-double-billed.md
|
package/src/agents/shared.ts
CHANGED
|
@@ -55,14 +55,25 @@ async function ensureInstalled(sandbox: Sandbox, cmd: string, args: string[]): P
|
|
|
55
55
|
* 用沙箱里的 node 递归找全局最新:不依赖 GNU find 的 -printf(BSD / 精简镜像没有),
|
|
56
56
|
* 也没有 `-exec ls -t +` 的 ARG_MAX 分批陷阱(每批各自排序,head -1 不是全局最新)。
|
|
57
57
|
* 沙箱必然有 node(agent CLI 靠 npm 装,预制镜像也带)。
|
|
58
|
+
*
|
|
59
|
+
* `excludeName` 可选:跳过文件名匹配的旁路产物(例如 OpenClaw 同目录下的
|
|
60
|
+
* `*.trajectory.jsonl`,mtime 常比 session transcript 新,但不是消息轨)。
|
|
58
61
|
*/
|
|
59
|
-
async function captureLatestJsonl(
|
|
62
|
+
async function captureLatestJsonl(
|
|
63
|
+
sandbox: Sandbox,
|
|
64
|
+
dir: string,
|
|
65
|
+
opts?: { excludeName?: RegExp },
|
|
66
|
+
): Promise<string | undefined> {
|
|
67
|
+
const excludeSource = opts?.excludeName?.source ?? "";
|
|
68
|
+
const excludeFlags = opts?.excludeName?.flags ?? "";
|
|
69
|
+
// exclude 正则以 JSON 字面量塞进 node -e,避免 shell 转义坑;空串 = 不排除。
|
|
60
70
|
const script =
|
|
61
71
|
'const fs=require("fs"),p=require("path");let best=null;' +
|
|
72
|
+
`const exclude=${JSON.stringify(excludeSource)}?new RegExp(${JSON.stringify(excludeSource)},${JSON.stringify(excludeFlags)}):null;` +
|
|
62
73
|
"const walk=(d)=>{let es;try{es=fs.readdirSync(d,{withFileTypes:true})}catch{return}" +
|
|
63
74
|
"for(const e of es){const f=p.join(d,e.name);" +
|
|
64
75
|
"if(e.isDirectory())walk(f);" +
|
|
65
|
-
'else if(e.name.endsWith(".jsonl")){try{const m=fs.statSync(f).mtimeMs;if(!best||m>best.m)best={f,m}}catch{}}}};' +
|
|
76
|
+
'else if(e.name.endsWith(".jsonl")&&!(exclude&&exclude.test(e.name))){try{const m=fs.statSync(f).mtimeMs;if(!best||m>best.m)best={f,m}}catch{}}}};' +
|
|
66
77
|
"walk(process.argv[1]);if(best)process.stdout.write(best.f);";
|
|
67
78
|
try {
|
|
68
79
|
// dir 不加引号以便 shell 展开 ~;受信内部路径(同 writeFile 的约定)。
|
package/src/agents/types.ts
CHANGED
|
@@ -75,7 +75,7 @@ export type AgentSetupSkill =
|
|
|
75
75
|
/**
|
|
76
76
|
* 一次 Agent setup 实际装了什么 —— 沙箱型 Coding Agent Adapter 在 setup 收尾写出的安装清单。
|
|
77
77
|
* 沙箱内落在 `__niceeval__/agent-setup.json`,运行器把它作为 attempt artifact 存成
|
|
78
|
-
* `agent-setup.json`(见 docs/feature/
|
|
78
|
+
* `agent-setup.json`(见 docs/feature/record/architecture.md)。不参与评分,只回答「这次实际
|
|
79
79
|
* 装了什么」;**环境变量值与 secret 不写进来**(所以 mcpServers 只记 name/command/args)。
|
|
80
80
|
*/
|
|
81
81
|
export interface AgentSetupManifest {
|
|
@@ -322,7 +322,7 @@ export interface AgentContext {
|
|
|
322
322
|
* 第三条反馈通道:上报本次运行的中性环境观测(如实际生效的 agent 配置、缓存命中状态)。
|
|
323
323
|
* 落进 `AttemptRecord.facts` 成为一等观测量;不影响 Turn status 或 verdict,不参与
|
|
324
324
|
* verdict / 评分 / 指纹。key 匹配 `[a-z0-9._-]{1,64}`,value 是标量;同 key 后写覆盖先写,
|
|
325
|
-
* 非法 key 或非标量 value 抛错。形状与归属语义见 docs/feature/
|
|
325
|
+
* 非法 key 或非标量 value 抛错。形状与归属语义见 docs/feature/record/architecture.md#facts运行事实。
|
|
326
326
|
*/
|
|
327
327
|
fact(key: string, value: string | number | boolean): void;
|
|
328
328
|
/**
|
|
@@ -270,7 +270,7 @@ describe("SessionManager · 多轮 Usage 累计", () => {
|
|
|
270
270
|
|
|
271
271
|
// turn 级 usage 挂接:该轮 Turn.usage 经 onTurn 回报给 runner,由 runner 挂上 TimingNode
|
|
272
272
|
// (src/runner/attempt.ts 的 onTurn → recorder.child({ usage })),show `--execution`/
|
|
273
|
-
// `--timing` 的 turn 头行读 TimingNode.usage(见 docs/feature/
|
|
273
|
+
// `--timing` 的 turn 头行读 TimingNode.usage(见 docs/feature/record/architecture.md
|
|
274
274
|
// 「result.json」TimingNode.usage,src/show/render.ts 的 turnUsageText)。这里只测
|
|
275
275
|
// session.ts 这一端:onTurn 收到的 usage 与该轮 Turn.usage 同值;没有 usage 的轮不传该字段。
|
|
276
276
|
describe("SessionManager · onTurn 回报的 usage(turn 挂接 TimingNode 的数据来源)", () => {
|
package/src/context/session.ts
CHANGED
|
@@ -438,7 +438,7 @@ export function lastAssistantText(events: readonly StreamEvent[]): string | unde
|
|
|
438
438
|
|
|
439
439
|
/**
|
|
440
440
|
* 每个字段只在某一轮真的带回该值时才累加,协议不提供就保持省略(见
|
|
441
|
-
* docs/feature/
|
|
441
|
+
* docs/feature/record/architecture.md「Usage」:「每个字段只在协议真实提供该值时存在……
|
|
442
442
|
* 不存在『默认 0』或『默认 1』的字段」)。此前 `requests` 用 `add.requests ?? 1` 累加,会让
|
|
443
443
|
* 转录解析型 adapter(整个 attempt 只在末尾解析一次 transcript、天然不报每轮请求数)的一轮
|
|
444
444
|
* send 被硬算成 1 个请求,一个内部发起了 21 次工具调用的 codex session 因此落盘
|
package/src/o11y/cost.ts
CHANGED
|
@@ -57,7 +57,7 @@ function lookupBuiltin(model: string): Price | undefined {
|
|
|
57
57
|
|
|
58
58
|
/**
|
|
59
59
|
* 按 token 桶 × 单价估算一次运行的美元成本。逐桶相加成立的前提是 Usage 桶恒互斥
|
|
60
|
-
* (inputTokens 不含 cache 命中,归一义务在 adapter,见 docs/feature/
|
|
60
|
+
* (inputTokens 不含 cache 命中,归一义务在 adapter,见 docs/feature/record/architecture.md#usage);
|
|
61
61
|
* cache 桶缺专门单价时退回 input 价——宁可高估,不静默低估。
|
|
62
62
|
* 无 model / 查不到价(用户表 + 内置快照都没有)→ undefined。
|
|
63
63
|
*/
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// cases: docs/engineering/testing/unit/
|
|
1
|
+
// cases: docs/engineering/testing/unit/record.md
|
|
2
2
|
// 「Usage、facts 与失败命令证据落盘」桶恒互斥归一:bub tape 的 usage 是 Chat Completions
|
|
3
3
|
// 形状,prompt_tokens_details.cached_tokens 是 prompt_tokens 子集,聚合前扣减;
|
|
4
4
|
// 同对象的 cost 是实测计费,照常累进 costUSD。
|
package/src/o11y/parsers/bub.ts
CHANGED
|
@@ -237,7 +237,7 @@ export function parseBubTranscript(raw: string | undefined): ParsedTranscript {
|
|
|
237
237
|
}
|
|
238
238
|
|
|
239
239
|
// requests > 0 意味着至少一次 addUsage 真的读到了非零字段;整份 tape 没有任何 usage 时
|
|
240
|
-
// input/output 也不该垫成 0(见 docs/feature/
|
|
240
|
+
// input/output 也不该垫成 0(见 docs/feature/record/architecture.md#usage)。
|
|
241
241
|
const usage: Usage = requests > 0 ? { inputTokens, outputTokens } : {};
|
|
242
242
|
if (cacheReadTokens > 0) usage.cacheReadTokens = cacheReadTokens;
|
|
243
243
|
if (requests > 0) usage.requests = requests;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// cases: docs/engineering/testing/unit/
|
|
1
|
+
// cases: docs/engineering/testing/unit/record.md
|
|
2
2
|
// 「Usage、facts 与失败命令证据落盘」桶恒互斥归一:codex transcript 的 cached_input_tokens
|
|
3
3
|
// 是 input_tokens 子集,聚合前逐请求扣减。
|
|
4
4
|
// bug: memory/estimatecost-openai-inclusive-cache-double-billed.md
|
|
@@ -419,7 +419,7 @@ export function parseCodexTranscript(raw: string | undefined): ParsedTranscript
|
|
|
419
419
|
}
|
|
420
420
|
|
|
421
421
|
// requests > 0 意味着至少一行真的带回了 usage;整份 transcript 没有任何 usage 行时
|
|
422
|
-
// input/output 也不该垫成 0(见 docs/feature/
|
|
422
|
+
// input/output 也不该垫成 0(见 docs/feature/record/architecture.md#usage)。
|
|
423
423
|
const usage: Usage = requests > 0 ? { inputTokens, outputTokens } : {};
|
|
424
424
|
if (cacheReadTokens > 0) usage.cacheReadTokens = cacheReadTokens;
|
|
425
425
|
if (reasoningTokens > 0) usage.reasoningTokens = reasoningTokens;
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
// Hermes sessions export / state.db 消息行解析器。方言只住这里,不进 core
|
|
2
|
+
// (契约见 docs/feature/adapters/sdk/hermes/README.md)。
|
|
3
|
+
|
|
4
|
+
import type { StreamEvent, Usage, ToolName, JsonValue } from "../../types.ts";
|
|
5
|
+
import type { ParsedTranscript } from "./index.ts";
|
|
6
|
+
import { GENERIC_VERB_ALIASES, normalizeToolName as normalizeShared } from "../tool-names.ts";
|
|
7
|
+
|
|
8
|
+
export const HERMES_TOOL_ALIASES: Record<string, ToolName> = {
|
|
9
|
+
...GENERIC_VERB_ALIASES,
|
|
10
|
+
terminal: "shell",
|
|
11
|
+
read_file: "file_read",
|
|
12
|
+
write_file: "file_write",
|
|
13
|
+
edit_file: "file_edit",
|
|
14
|
+
web_extract: "web_fetch",
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
function normalizeToolName(name: string): ToolName {
|
|
18
|
+
return normalizeShared(name, HERMES_TOOL_ALIASES);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function get(obj: unknown, key: string): unknown {
|
|
22
|
+
return obj && typeof obj === "object" ? (obj as Record<string, unknown>)[key] : undefined;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function str(v: unknown): string | undefined {
|
|
26
|
+
return typeof v === "string" && v ? v : undefined;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function num(obj: unknown, ...keys: string[]): number {
|
|
30
|
+
for (const k of keys) {
|
|
31
|
+
const v = get(obj, k);
|
|
32
|
+
if (typeof v === "number" && Number.isFinite(v)) return v;
|
|
33
|
+
}
|
|
34
|
+
return 0;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function coerceArgs(value: unknown): JsonValue {
|
|
38
|
+
if (typeof value === "string") {
|
|
39
|
+
try {
|
|
40
|
+
return JSON.parse(value) as JsonValue;
|
|
41
|
+
} catch {
|
|
42
|
+
return value;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return (value ?? {}) as JsonValue;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function parseJsonField(value: unknown): unknown {
|
|
49
|
+
if (typeof value !== "string") return value;
|
|
50
|
+
try {
|
|
51
|
+
return JSON.parse(value);
|
|
52
|
+
} catch {
|
|
53
|
+
return value;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Hermes sessions export JSONL(一行一 session,或一行一 message)→ 标准事件 + usage。
|
|
59
|
+
*/
|
|
60
|
+
export function parseHermesTranscript(raw: string | undefined): ParsedTranscript {
|
|
61
|
+
const events: StreamEvent[] = [];
|
|
62
|
+
let inputTokens = 0;
|
|
63
|
+
let outputTokens = 0;
|
|
64
|
+
let cacheReadTokens = 0;
|
|
65
|
+
let cacheCreationTokens = 0;
|
|
66
|
+
let costUSD = 0;
|
|
67
|
+
let requests = 0;
|
|
68
|
+
let parseSuccess = true;
|
|
69
|
+
let synth = 0;
|
|
70
|
+
const nextSynthId = (): string => `hm_${++synth}`;
|
|
71
|
+
|
|
72
|
+
if (!raw?.trim()) {
|
|
73
|
+
return { events, usage: {}, compactions: 0, parseSuccess: true };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const ingestSessionMeta = (obj: Record<string, unknown>): void => {
|
|
77
|
+
inputTokens += num(obj, "input_tokens", "inputTokens");
|
|
78
|
+
outputTokens += num(obj, "output_tokens", "outputTokens");
|
|
79
|
+
cacheReadTokens += num(obj, "cache_read_tokens", "cacheReadTokens");
|
|
80
|
+
cacheCreationTokens += num(obj, "cache_write_tokens", "cacheWriteTokens", "cacheCreationTokens");
|
|
81
|
+
const actual = num(obj, "actual_cost_usd", "actualCostUsd");
|
|
82
|
+
const estimated = num(obj, "estimated_cost_usd", "estimatedCostUsd");
|
|
83
|
+
costUSD += actual || estimated;
|
|
84
|
+
const apiCalls = num(obj, "api_call_count", "apiCallCount");
|
|
85
|
+
if (apiCalls) requests += apiCalls;
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
const ingestMessage = (msg: Record<string, unknown>): void => {
|
|
89
|
+
const role = str(msg.role) ?? "assistant";
|
|
90
|
+
const content = str(msg.content);
|
|
91
|
+
|
|
92
|
+
if (role === "tool" || str(msg.tool_call_id) || str(msg.toolCallId)) {
|
|
93
|
+
const callId = str(msg.tool_call_id) ?? str(msg.toolCallId) ?? nextSynthId();
|
|
94
|
+
const success = msg.is_error !== true && msg.isError !== true;
|
|
95
|
+
events.push({
|
|
96
|
+
type: "action.result",
|
|
97
|
+
callId,
|
|
98
|
+
output: (content ?? get(msg, "output") ?? null) as JsonValue,
|
|
99
|
+
status: success ? "completed" : "failed",
|
|
100
|
+
});
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const toolCallsRaw = parseJsonField(msg.tool_calls ?? msg.toolCalls);
|
|
105
|
+
if (Array.isArray(toolCallsRaw)) {
|
|
106
|
+
for (const call of toolCallsRaw) {
|
|
107
|
+
const fn = get(call, "function") as Record<string, unknown> | undefined;
|
|
108
|
+
const name =
|
|
109
|
+
str(get(fn, "name")) ?? str(get(call, "name")) ?? str(msg.tool_name) ?? "unknown";
|
|
110
|
+
const callId = str(get(call, "id")) ?? nextSynthId();
|
|
111
|
+
const args = coerceArgs(get(fn, "arguments") ?? get(call, "arguments") ?? get(call, "input"));
|
|
112
|
+
events.push({
|
|
113
|
+
type: "action.called",
|
|
114
|
+
callId,
|
|
115
|
+
name,
|
|
116
|
+
input: args,
|
|
117
|
+
tool: normalizeToolName(name),
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
if (content?.trim() && role !== "user" && role !== "system") {
|
|
123
|
+
events.push({ type: "message", role: "assistant", text: content });
|
|
124
|
+
} else if (content?.trim() && role === "user" && !Array.isArray(toolCallsRaw)) {
|
|
125
|
+
// 用户消息不进行为断言主路径;保留会污染 messageIncludes。跳过。
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const reasoning = str(msg.reasoning) ?? str(msg.reasoning_content);
|
|
129
|
+
if (reasoning?.trim()) events.push({ type: "thinking", text: reasoning });
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
for (const line of raw.split("\n")) {
|
|
133
|
+
const trimmed = line.trim();
|
|
134
|
+
if (!trimmed) continue;
|
|
135
|
+
let obj: Record<string, unknown>;
|
|
136
|
+
try {
|
|
137
|
+
obj = JSON.parse(trimmed) as Record<string, unknown>;
|
|
138
|
+
} catch {
|
|
139
|
+
parseSuccess = false;
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
try {
|
|
144
|
+
// 整 session 导出:带 messages[]
|
|
145
|
+
if (Array.isArray(obj.messages)) {
|
|
146
|
+
ingestSessionMeta(obj);
|
|
147
|
+
for (const m of obj.messages) {
|
|
148
|
+
if (m && typeof m === "object") ingestMessage(m as Record<string, unknown>);
|
|
149
|
+
}
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
// 单行 message
|
|
153
|
+
if (str(obj.role)) {
|
|
154
|
+
ingestMessage(obj);
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
// sqlite dump 行:{session_id, role, content, tool_calls, ...}
|
|
158
|
+
if (str(obj.session_id) || str(obj.sessionId)) {
|
|
159
|
+
ingestMessage(obj);
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
ingestSessionMeta(obj);
|
|
163
|
+
} catch {
|
|
164
|
+
parseSuccess = false;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const usage: Usage = {};
|
|
169
|
+
if (inputTokens) usage.inputTokens = inputTokens;
|
|
170
|
+
if (outputTokens) usage.outputTokens = outputTokens;
|
|
171
|
+
if (cacheReadTokens) usage.cacheReadTokens = cacheReadTokens;
|
|
172
|
+
if (cacheCreationTokens) usage.cacheCreationTokens = cacheCreationTokens;
|
|
173
|
+
if (costUSD) usage.costUSD = costUSD;
|
|
174
|
+
if (requests) usage.requests = requests;
|
|
175
|
+
|
|
176
|
+
return { events, usage, compactions: 0, parseSuccess };
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** 从 export / stdout 文本里抠 session id。 */
|
|
180
|
+
export function sessionIdFromHermesOutput(raw: string | undefined): string | undefined {
|
|
181
|
+
if (!raw) return undefined;
|
|
182
|
+
for (const line of raw.split("\n")) {
|
|
183
|
+
const resume = line.match(/hermes\s+(?:chat\s+)?(?:--resume|-r)\s+(\S+)/i);
|
|
184
|
+
if (resume?.[1]) return resume[1].replace(/["'`]/g, "");
|
|
185
|
+
const labeled = line.match(/Session:\s*([A-Za-z0-9_-]+)/i);
|
|
186
|
+
if (labeled?.[1]) return labeled[1];
|
|
187
|
+
const trimmed = line.trim();
|
|
188
|
+
if (!trimmed.startsWith("{")) continue;
|
|
189
|
+
try {
|
|
190
|
+
const obj = JSON.parse(trimmed) as Record<string, unknown>;
|
|
191
|
+
const id = str(obj.id) ?? str(obj.session_id) ?? str(obj.sessionId);
|
|
192
|
+
if (id && (Array.isArray(obj.messages) || str(obj.source))) return id;
|
|
193
|
+
} catch {
|
|
194
|
+
// skip
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
return undefined;
|
|
198
|
+
}
|
|
@@ -22,6 +22,7 @@ export const OPENCLAW_TOOL_ALIASES: Record<string, ToolName> = {
|
|
|
22
22
|
read: "file_read",
|
|
23
23
|
write: "file_write",
|
|
24
24
|
edit: "file_edit",
|
|
25
|
+
exec: "shell",
|
|
25
26
|
process: "shell",
|
|
26
27
|
browser: "web_fetch",
|
|
27
28
|
};
|
|
@@ -203,7 +204,7 @@ export function parseOpenClawTranscript(raw: string | undefined): ParsedTranscri
|
|
|
203
204
|
}
|
|
204
205
|
|
|
205
206
|
// requests > 0 意味着至少一条消息真的带回了 usage;整份 transcript 没有任何 usage 时
|
|
206
|
-
// input/output 也不该垫成 0(见 docs/feature/
|
|
207
|
+
// input/output 也不该垫成 0(见 docs/feature/record/architecture.md#usage)。
|
|
207
208
|
const usage: Usage = requests > 0 ? { inputTokens, outputTokens } : {};
|
|
208
209
|
if (cacheReadTokens > 0) usage.cacheReadTokens = cacheReadTokens;
|
|
209
210
|
if (cacheCreationTokens > 0) usage.cacheCreationTokens = cacheCreationTokens;
|
|
@@ -224,6 +225,11 @@ export interface OpenClawRunJson {
|
|
|
224
225
|
text?: string;
|
|
225
226
|
/** 本次 run 的 session key(首轮 capture、后续 resume 用)。 */
|
|
226
227
|
sessionId?: string;
|
|
228
|
+
/**
|
|
229
|
+
* 本轮 session transcript 绝对路径(`meta.agentMeta.sessionFile`)。
|
|
230
|
+
* 有的话优先按路径读,避开同目录 `*.trajectory.jsonl` 抢「最新」mtime。
|
|
231
|
+
*/
|
|
232
|
+
sessionFile?: string;
|
|
227
233
|
/** 封包报告的用量;没报就省略,不编造。 */
|
|
228
234
|
usage?: Usage;
|
|
229
235
|
/** 封包自报失败(error 字段 / status=error|failed / success=false)。 */
|
|
@@ -253,14 +259,29 @@ export function parseOpenClawRunJson(stdout: string | undefined): OpenClawRunJso
|
|
|
253
259
|
str(get(doc, "reply")) ??
|
|
254
260
|
str(get(doc, "message"));
|
|
255
261
|
|
|
262
|
+
const meta = get(doc, "meta");
|
|
263
|
+
const agentMeta = get(meta, "agentMeta");
|
|
264
|
+
|
|
256
265
|
const sessionId =
|
|
257
266
|
str(get(doc, "sessionId")) ??
|
|
258
267
|
str(get(doc, "session_id")) ??
|
|
259
268
|
str(get(doc, "sessionKey")) ??
|
|
260
269
|
str(get(result, "sessionId")) ??
|
|
261
|
-
str(get(result, "session_id"))
|
|
270
|
+
str(get(result, "session_id")) ??
|
|
271
|
+
str(get(agentMeta, "sessionId")) ??
|
|
272
|
+
str(get(agentMeta, "session_id"));
|
|
273
|
+
|
|
274
|
+
const sessionFile =
|
|
275
|
+
str(get(agentMeta, "sessionFile")) ??
|
|
276
|
+
str(get(doc, "sessionFile")) ??
|
|
277
|
+
str(get(result, "sessionFile"));
|
|
262
278
|
|
|
263
|
-
const rawUsage =
|
|
279
|
+
const rawUsage =
|
|
280
|
+
get(doc, "usage") ??
|
|
281
|
+
get(result, "usage") ??
|
|
282
|
+
get(meta, "usage") ??
|
|
283
|
+
get(agentMeta, "usage") ??
|
|
284
|
+
get(agentMeta, "lastCallUsage");
|
|
264
285
|
let usage: Usage | undefined;
|
|
265
286
|
if (rawUsage && typeof rawUsage === "object") {
|
|
266
287
|
const input = num(rawUsage, "input", "input_tokens", "inputTokens", "prompt_tokens");
|
|
@@ -283,6 +304,7 @@ export function parseOpenClawRunJson(stdout: string | undefined): OpenClawRunJso
|
|
|
283
304
|
return {
|
|
284
305
|
...(text !== undefined ? { text } : {}),
|
|
285
306
|
...(sessionId !== undefined ? { sessionId } : {}),
|
|
307
|
+
...(sessionFile !== undefined ? { sessionFile } : {}),
|
|
286
308
|
...(usage !== undefined ? { usage } : {}),
|
|
287
309
|
failed,
|
|
288
310
|
};
|