niceeval 0.11.1 → 0.11.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/INDEX.md +1 -0
  2. package/README.md +4 -2
  3. package/README.zh.md +4 -2
  4. package/dist/agents/types.d.ts +2 -2
  5. package/dist/o11y/types.d.ts +3 -3
  6. package/dist/report/assets/colors.d.ts +1 -1
  7. package/dist/report/assets/colors.js +1 -1
  8. package/dist/report/components/attempt-detail/AttemptSource.js +1 -2
  9. package/dist/report/components/attempt-detail/compute.d.ts +4 -4
  10. package/dist/report/components/attempt-detail/compute.js +6 -6
  11. package/dist/report/components/attempt-detail/faces.d.ts +1 -1
  12. package/dist/report/components/attempt-detail/faces.js +3 -3
  13. package/dist/report/components/entity-lists/AttemptList.d.ts +3 -3
  14. package/dist/report/components/entity-lists/AttemptList.js +3 -3
  15. package/dist/report/components/entity-lists/ExperimentList.js +2 -2
  16. package/dist/report/components/entity-lists/compute.js +1 -1
  17. package/dist/report/components/entity-lists/faces.js +1 -1
  18. package/dist/report/components/entity-lists/index.d.ts +1 -1
  19. package/dist/report/components/entity-lists/index.js +1 -1
  20. package/dist/report/components/metric-views/chart-math.d.ts +2 -2
  21. package/dist/report/components/metric-views/chart-math.js +2 -2
  22. package/dist/report/components/metric-views/compute.d.ts +2 -3
  23. package/dist/report/components/metric-views/compute.js +3 -4
  24. package/dist/report/components/shared.d.ts +1 -1
  25. package/dist/report/components/site-components/compute.d.ts +5 -5
  26. package/dist/report/components/site-components/compute.js +6 -6
  27. package/dist/report/components/site-components/index.d.ts +7 -7
  28. package/dist/report/components/site-components/index.js +7 -7
  29. package/dist/report/components/site-components/scope-warnings.d.ts +1 -1
  30. package/dist/report/components/site-components/scope-warnings.js +3 -3
  31. package/dist/report/components/site-components/snapshot-diagnostics.js +1 -1
  32. package/dist/report/components/summaries/compute.d.ts +1 -1
  33. package/dist/report/components/summaries/compute.js +2 -2
  34. package/dist/report/components/summaries/faces.js +1 -1
  35. package/dist/report/components/summaries/index.d.ts +1 -1
  36. package/dist/report/components/summaries/index.js +1 -1
  37. package/dist/report/index.js +1 -1
  38. package/dist/report/model/aggregate.d.ts +2 -2
  39. package/dist/report/model/aggregate.js +2 -2
  40. package/dist/report/model/format.d.ts +2 -2
  41. package/dist/report/model/format.js +2 -2
  42. package/dist/report/model/metrics.js +3 -3
  43. package/dist/report/model/types.d.ts +11 -12
  44. package/dist/results/locator.js +1 -1
  45. package/dist/results/select.d.ts +4 -4
  46. package/dist/results/select.js +5 -5
  47. package/dist/results/types.d.ts +8 -8
  48. package/dist/runner/types.d.ts +14 -14
  49. package/dist/sandbox/resolve.d.ts +1 -1
  50. package/dist/sandbox/types.d.ts +1 -1
  51. package/docs-site/zh/reference/builtin-agents.mdx +21 -0
  52. package/docs-site/zh/reference/define-agent.mdx +1 -1
  53. package/docs-site/zh/reference/report-components.mdx +156 -74
  54. package/docs-site/zh/tutorials/custom-reports.mdx +112 -22
  55. package/docs-site/zh/tutorials/sandbox-agent.mdx +3 -4
  56. package/docs-site/zh/tutorials/sandbox-providers.mdx +15 -15
  57. package/docs-site/zh/tutorials/theming.mdx +169 -0
  58. package/package.json +2 -1
  59. package/src/agents/ai-sdk.test.ts +1 -1
  60. package/src/agents/bub-install-spec.ts +26 -8
  61. package/src/agents/bub.ts +40 -15
  62. package/src/agents/builtin.ts +11 -1
  63. package/src/agents/coding-cli-versions.ts +67 -0
  64. package/src/agents/hermes.ts +235 -0
  65. package/src/agents/index.ts +4 -0
  66. package/src/agents/langgraph.test.ts +1 -1
  67. package/src/agents/manifest.ts +1 -1
  68. package/src/agents/openai-compat.test.ts +1 -1
  69. package/src/agents/openclaw.ts +103 -24
  70. package/src/agents/opencode.ts +183 -0
  71. package/src/agents/sdk-streams.test.ts +1 -1
  72. package/src/agents/shared.ts +13 -2
  73. package/src/agents/types.ts +2 -2
  74. package/src/context/session.test.ts +1 -1
  75. package/src/context/session.ts +1 -1
  76. package/src/o11y/cost.ts +1 -1
  77. package/src/o11y/parsers/bub.test.ts +1 -1
  78. package/src/o11y/parsers/bub.ts +1 -1
  79. package/src/o11y/parsers/codex.test.ts +1 -1
  80. package/src/o11y/parsers/codex.ts +1 -1
  81. package/src/o11y/parsers/hermes.ts +198 -0
  82. package/src/o11y/parsers/openclaw.ts +25 -3
  83. package/src/o11y/parsers/opencode.ts +295 -0
  84. package/src/o11y/types.ts +3 -3
  85. package/src/report/assets/colors.ts +1 -1
  86. package/src/report/assets/styles.css +17 -15
  87. package/src/report/components/attempt-detail/AttemptAssertions.tsx +1 -1
  88. package/src/report/components/attempt-detail/AttemptConversation.tsx +1 -1
  89. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +1 -1
  90. package/src/report/components/attempt-detail/AttemptDiff.tsx +1 -1
  91. package/src/report/components/attempt-detail/AttemptError.tsx +1 -1
  92. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +1 -1
  93. package/src/report/components/attempt-detail/AttemptSource.tsx +2 -3
  94. package/src/report/components/attempt-detail/AttemptSummary.tsx +1 -1
  95. package/src/report/components/attempt-detail/AttemptTimeline.tsx +1 -1
  96. package/src/report/components/attempt-detail/AttemptTrace.tsx +1 -1
  97. package/src/report/components/attempt-detail/UsageTable.tsx +1 -1
  98. package/src/report/components/attempt-detail/attempt-components.test.tsx +1 -1
  99. package/src/report/components/attempt-detail/compute.ts +6 -6
  100. package/src/report/components/attempt-detail/faces.ts +3 -3
  101. package/src/report/components/attempt-detail/index.tsx +1 -1
  102. package/src/report/components/entity-lists/AttemptList.tsx +3 -3
  103. package/src/report/components/entity-lists/ExperimentList.tsx +2 -2
  104. package/src/report/components/entity-lists/compute.ts +1 -1
  105. package/src/report/components/entity-lists/faces.ts +1 -1
  106. package/src/report/components/entity-lists/index.tsx +1 -1
  107. package/src/report/components/metric-views/DeltaTable.tsx +1 -2
  108. package/src/report/components/metric-views/StabilityMatrix.tsx +1 -1
  109. package/src/report/components/metric-views/chart-math.test.ts +1 -1
  110. package/src/report/components/metric-views/chart-math.ts +2 -2
  111. package/src/report/components/metric-views/compute.ts +3 -4
  112. package/src/report/components/shared.ts +1 -1
  113. package/src/report/components/site-components/CopyFixPrompt.tsx +1 -1
  114. package/src/report/components/site-components/HeroCard.tsx +1 -1
  115. package/src/report/components/site-components/PoweredBy.tsx +1 -1
  116. package/src/report/components/site-components/ScopeWarnings.tsx +1 -1
  117. package/src/report/components/site-components/SnapshotDiagnostics.tsx +1 -1
  118. package/src/report/components/site-components/TraceWaterfall.tsx +1 -1
  119. package/src/report/components/site-components/compute.ts +6 -6
  120. package/src/report/components/site-components/index.tsx +7 -7
  121. package/src/report/components/site-components/scope-warnings.ts +3 -3
  122. package/src/report/components/site-components/snapshot-diagnostics.ts +1 -1
  123. package/src/report/components/summaries/ScopeSummary.tsx +2 -2
  124. package/src/report/components/summaries/compute.ts +2 -2
  125. package/src/report/components/summaries/faces.ts +1 -1
  126. package/src/report/components/summaries/index.tsx +1 -1
  127. package/src/report/index.ts +1 -1
  128. package/src/report/model/aggregate.ts +2 -2
  129. package/src/report/model/format.ts +2 -2
  130. package/src/report/model/metrics.ts +3 -3
  131. package/src/report/model/types.ts +12 -13
  132. package/src/results/annotated-source.test.ts +1 -1
  133. package/src/results/attempt-evidence.test.ts +1 -1
  134. package/src/results/copy.ts +3 -3
  135. package/src/results/format.ts +3 -3
  136. package/src/results/host-equivalence.test.ts +6 -6
  137. package/src/results/index.ts +1 -1
  138. package/src/results/locator.test.ts +2 -2
  139. package/src/results/locator.ts +1 -1
  140. package/src/results/open.ts +2 -2
  141. package/src/results/publish.ts +1 -1
  142. package/src/results/results.test.ts +6 -323
  143. package/src/results/select.test.ts +438 -0
  144. package/src/results/select.ts +6 -6
  145. package/src/results/skipped-notice.ts +1 -1
  146. package/src/results/truncate.ts +2 -2
  147. package/src/results/types.ts +9 -9
  148. package/src/results/writer.ts +4 -4
  149. package/src/runner/attempt.test.ts +2 -2
  150. package/src/runner/attempt.ts +6 -6
  151. package/src/runner/eval-source.test.ts +1 -1
  152. package/src/runner/eval-source.ts +1 -1
  153. package/src/runner/reporters/artifacts.ts +1 -1
  154. package/src/runner/run.ts +2 -2
  155. package/src/runner/timing.ts +1 -1
  156. package/src/runner/types.ts +14 -14
  157. package/src/sandbox/docker-agent-image.ts +36 -0
  158. package/src/sandbox/e2b-agent-template.test.ts +6 -21
  159. package/src/sandbox/e2b-agent-template.ts +44 -11
  160. package/src/sandbox/index.ts +8 -0
  161. package/src/sandbox/official-baselines.test.ts +175 -0
  162. package/src/sandbox/resolve.ts +1 -1
  163. package/src/sandbox/types.ts +1 -1
  164. package/src/scoring/diff.ts +1 -1
  165. package/src/scoring/types.ts +1 -1
  166. package/src/shared/facts.ts +1 -1
  167. package/src/show/index.ts +1 -1
  168. package/src/show/render.ts +1 -1
  169. package/src/show/show.test.ts +3 -3
  170. package/src/view/app/App.tsx +3 -1
  171. package/src/view/client-dist/app.css +1 -1
  172. package/src/view/client-dist/app.js +1 -1
  173. package/src/view/data.test.ts +2 -2
  174. package/src/view/data.ts +1 -1
  175. package/src/view/site-base.test.ts +51 -0
  176. package/src/view/site.ts +16 -0
  177. package/src/view/styles.css +1 -2
  178. package/src/view/template.html +1 -0
  179. package/src/view/view-report.test.ts +1 -1
@@ -0,0 +1,183 @@
1
+ import { defineSandboxAgent } from "../define.ts";
2
+ import { requireEnv, getEnv } from "../util.ts";
3
+ import { shared } from "./shared.ts";
4
+ import {
5
+ appendProjectInstruction,
6
+ installSkills,
7
+ installedSkillNames,
8
+ skillDiscoveryInstruction,
9
+ } from "./skills.ts";
10
+ import { writeAgentSetupManifest } from "./manifest.ts";
11
+ import { mapGenericSpans } from "../o11y/otlp/canonical.ts";
12
+ import { completeCoverage } from "../scoring/coverage.ts";
13
+ import {
14
+ parseOpenCodeTranscript,
15
+ sessionIdFromOpenCodeTranscript,
16
+ extractOpenCodeJsonl,
17
+ } from "../o11y/parsers/opencode.ts";
18
+ import { DEFAULT_OPENCODE_CLI_VERSION } from "./coding-cli-versions.ts";
19
+ import type { Agent, AgentSetupManifest, SkillSpec, StreamEvent } from "../types.ts";
20
+
21
+ // OpenCode sandbox adapter。驱动:`opencode run --format json --auto`;
22
+ // 行为轨优先 stdout JSONL,不足时 `opencode export <sessionID>`。
23
+ // 契约见 docs/feature/adapters/sdk/opencode/README.md。
24
+
25
+ const SKILL_DIR = ".agents/skills";
26
+ /** 自定义 OpenAI 兼容网关在 opencode.json 里的 provider id。 */
27
+ const COMPAT_PROVIDER = "compat";
28
+
29
+ export interface OpenCodeConfig {
30
+ /** 模型 API key。省略时读 OPENCODE_API_KEY,再回落 ANTHROPIC_API_KEY。 */
31
+ apiKey?: string;
32
+ /** OpenAI 兼容端点。省略时读 OPENCODE_BASE_URL。 */
33
+ baseUrl?: string;
34
+ /** 钉 npm `opencode-ai` 版本;省略时用 NiceEval 默认 pin。 */
35
+ version?: string;
36
+ /** 装进沙箱的 Skill,落在 `.agents/skills/<name>/`。 */
37
+ skills?: SkillSpec[];
38
+ }
39
+
40
+ function resolveApiKey(config?: OpenCodeConfig): string {
41
+ return config?.apiKey ?? getEnv("OPENCODE_API_KEY") ?? requireEnv("ANTHROPIC_API_KEY");
42
+ }
43
+
44
+ function resolveBaseUrl(config?: OpenCodeConfig): string | undefined {
45
+ return config?.baseUrl ?? getEnv("OPENCODE_BASE_URL");
46
+ }
47
+
48
+ /** experiment.model → `--model`。裸模型名在自定义网关下补成 `compat/<model>`。 */
49
+ function resolveModelFlag(model: string | undefined, hasCompatBase: boolean): string | undefined {
50
+ if (!model) return undefined;
51
+ if (model.includes("/")) return model;
52
+ return hasCompatBase ? `${COMPAT_PROVIDER}/${model}` : model;
53
+ }
54
+
55
+ /**
56
+ * OpenCode 的内置 sandbox Agent 工厂。
57
+ */
58
+ export function openCodeAgent(config?: OpenCodeConfig): Agent {
59
+ const version = config?.version ?? DEFAULT_OPENCODE_CLI_VERSION;
60
+
61
+ return defineSandboxAgent({
62
+ name: "opencode",
63
+ coverage: completeCoverage,
64
+ spanMapper: mapGenericSpans,
65
+
66
+ tracing: {
67
+ protocol: "http/protobuf",
68
+ env: (endpoint) => ({
69
+ OTEL_TRACES_EXPORTER: "otlp",
70
+ OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: endpoint,
71
+ OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: "http/protobuf",
72
+ }),
73
+ },
74
+
75
+ async setup(sb) {
76
+ await sb.runShell(
77
+ `command -v opencode >/dev/null 2>&1 || npm install -g opencode-ai@${version}`,
78
+ );
79
+
80
+ const baseUrl = resolveBaseUrl(config);
81
+ const provider: Record<string, unknown> = {};
82
+ if (baseUrl) {
83
+ // OpenAI 兼容网关必须走 openai-compatible npm 包 + 显式 models 表,
84
+ // 否则 OpenCode 不会把自定义 baseURL 接到请求上。
85
+ provider[COMPAT_PROVIDER] = {
86
+ npm: "@ai-sdk/openai-compatible",
87
+ name: "OpenAI-compatible",
88
+ options: {
89
+ apiKey: "{env:OPENCODE_API_KEY}",
90
+ baseURL: baseUrl,
91
+ },
92
+ models: {
93
+ // 允许任意 experiment.model;具体 id 由 --model compat/<id> 选择。
94
+ "gpt-5.6-luna": { name: "gpt-5.6-luna" },
95
+ "gpt-5.4-mini": { name: "gpt-5.4-mini" },
96
+ "gpt-5.4": { name: "gpt-5.4" },
97
+ "gpt-4.1-mini": { name: "gpt-4.1-mini" },
98
+ "gpt-4o-mini": { name: "gpt-4o-mini" },
99
+ },
100
+ };
101
+ }
102
+ const opencodeConfig = {
103
+ $schema: "https://opencode.ai/config.json",
104
+ ...(Object.keys(provider).length ? { provider } : {}),
105
+ permission: {
106
+ write: "allow",
107
+ edit: "allow",
108
+ bash: "allow",
109
+ read: "allow",
110
+ },
111
+ };
112
+ await shared.writeFile(sb, "opencode.json", JSON.stringify(opencodeConfig, null, 2));
113
+
114
+ const manifest: AgentSetupManifest = { skills: [] };
115
+ if (config?.skills?.length) {
116
+ manifest.skills = await installSkills(sb, config.skills, { dir: SKILL_DIR });
117
+ await appendProjectInstruction(
118
+ sb,
119
+ skillDiscoveryInstruction(SKILL_DIR, installedSkillNames(manifest.skills)),
120
+ );
121
+ }
122
+ if (manifest.skills.length) {
123
+ await writeAgentSetupManifest(sb, manifest);
124
+ }
125
+ },
126
+
127
+ async send(input, ctx) {
128
+ const sb = ctx.sandbox;
129
+ const baseUrl = resolveBaseUrl(config);
130
+ const args = ["run", input.text, "--format", "json", "--auto"];
131
+ const model = resolveModelFlag(ctx.model, Boolean(baseUrl));
132
+ if (model) args.push("--model", model);
133
+ if (ctx.session.id) args.push("--session", ctx.session.id);
134
+
135
+ const apiKey = resolveApiKey(config);
136
+ const env: Record<string, string> = {
137
+ OPENCODE_API_KEY: apiKey,
138
+ OPENAI_API_KEY: apiKey,
139
+ ANTHROPIC_API_KEY: apiKey,
140
+ ...ctx.telemetry?.env,
141
+ };
142
+ if (baseUrl) {
143
+ env.OPENCODE_BASE_URL = baseUrl;
144
+ env.OPENAI_BASE_URL = baseUrl;
145
+ }
146
+
147
+ const res = await sb.runCommand("opencode", args, { env, stream: true });
148
+ let raw = extractOpenCodeJsonl(res.stdout) ?? extractOpenCodeJsonl(`${res.stdout}\n${res.stderr}`);
149
+ let sessionId = sessionIdFromOpenCodeTranscript(raw) ?? sessionIdFromOpenCodeTranscript(res.stdout);
150
+ if (sessionId) ctx.session.capture(sessionId);
151
+
152
+ let parsed = parseOpenCodeTranscript(raw);
153
+ // 仅当 stdout 既没有工具也没有助手文本时才 export 补读——纯对话轮
154
+ // (session/recall) 的 text 事件已在 JSONL 里,再 export 会冲掉已解析事件。
155
+ const hasActions = parsed.events.some((e) => e.type === "action.called");
156
+ const hasMessages = parsed.events.some((e) => e.type === "message");
157
+ if (!hasActions && !hasMessages && (sessionId ?? ctx.session.id)) {
158
+ const sid = sessionId ?? ctx.session.id!;
159
+ const exported = await sb.runCommand("opencode", ["export", sid], { env });
160
+ if (exported.exitCode === 0 && exported.stdout.trim()) {
161
+ raw = exported.stdout;
162
+ parsed = parseOpenCodeTranscript(raw);
163
+ sessionId = sessionIdFromOpenCodeTranscript(raw) ?? sessionId;
164
+ if (sessionId) ctx.session.capture(sessionId);
165
+ }
166
+ }
167
+
168
+ const events: StreamEvent[] = [...parsed.events];
169
+ const hasErrorEvent = events.some((e) => e.type === "error");
170
+ const failed = res.exitCode !== 0 || hasErrorEvent;
171
+ if (res.exitCode !== 0) {
172
+ events.push({ type: "error", message: shared.diagnoseFailure(res, parsed.events, raw) });
173
+ }
174
+ return {
175
+ events,
176
+ usage: parsed.usage,
177
+ status: failed ? "failed" : "completed",
178
+ };
179
+ },
180
+ });
181
+ }
182
+
183
+ export default openCodeAgent();
@@ -1,4 +1,4 @@
1
- // cases: docs/engineering/testing/unit/results.md
1
+ // cases: docs/engineering/testing/unit/record.md
2
2
  // 「Usage、facts 与失败命令证据落盘」桶恒互斥归一:codex(OpenAI 口径,cached ⊂ input)扣减、
3
3
  // Anthropic / pi(互斥口径)如实转发。fixture 数值刻意让「扣与不扣」结果可区分。
4
4
  // bug: memory/estimatecost-openai-inclusive-cache-double-billed.md
@@ -55,14 +55,25 @@ async function ensureInstalled(sandbox: Sandbox, cmd: string, args: string[]): P
55
55
  * 用沙箱里的 node 递归找全局最新:不依赖 GNU find 的 -printf(BSD / 精简镜像没有),
56
56
  * 也没有 `-exec ls -t +` 的 ARG_MAX 分批陷阱(每批各自排序,head -1 不是全局最新)。
57
57
  * 沙箱必然有 node(agent CLI 靠 npm 装,预制镜像也带)。
58
+ *
59
+ * `excludeName` 可选:跳过文件名匹配的旁路产物(例如 OpenClaw 同目录下的
60
+ * `*.trajectory.jsonl`,mtime 常比 session transcript 新,但不是消息轨)。
58
61
  */
59
- async function captureLatestJsonl(sandbox: Sandbox, dir: string): Promise<string | undefined> {
62
+ async function captureLatestJsonl(
63
+ sandbox: Sandbox,
64
+ dir: string,
65
+ opts?: { excludeName?: RegExp },
66
+ ): Promise<string | undefined> {
67
+ const excludeSource = opts?.excludeName?.source ?? "";
68
+ const excludeFlags = opts?.excludeName?.flags ?? "";
69
+ // exclude 正则以 JSON 字面量塞进 node -e,避免 shell 转义坑;空串 = 不排除。
60
70
  const script =
61
71
  'const fs=require("fs"),p=require("path");let best=null;' +
72
+ `const exclude=${JSON.stringify(excludeSource)}?new RegExp(${JSON.stringify(excludeSource)},${JSON.stringify(excludeFlags)}):null;` +
62
73
  "const walk=(d)=>{let es;try{es=fs.readdirSync(d,{withFileTypes:true})}catch{return}" +
63
74
  "for(const e of es){const f=p.join(d,e.name);" +
64
75
  "if(e.isDirectory())walk(f);" +
65
- 'else if(e.name.endsWith(".jsonl")){try{const m=fs.statSync(f).mtimeMs;if(!best||m>best.m)best={f,m}}catch{}}}};' +
76
+ 'else if(e.name.endsWith(".jsonl")&&!(exclude&&exclude.test(e.name))){try{const m=fs.statSync(f).mtimeMs;if(!best||m>best.m)best={f,m}}catch{}}}};' +
66
77
  "walk(process.argv[1]);if(best)process.stdout.write(best.f);";
67
78
  try {
68
79
  // dir 不加引号以便 shell 展开 ~;受信内部路径(同 writeFile 的约定)。
@@ -75,7 +75,7 @@ export type AgentSetupSkill =
75
75
  /**
76
76
  * 一次 Agent setup 实际装了什么 —— 沙箱型 Coding Agent Adapter 在 setup 收尾写出的安装清单。
77
77
  * 沙箱内落在 `__niceeval__/agent-setup.json`,运行器把它作为 attempt artifact 存成
78
- * `agent-setup.json`(见 docs/feature/results/architecture.md)。不参与评分,只回答「这次实际
78
+ * `agent-setup.json`(见 docs/feature/record/architecture.md)。不参与评分,只回答「这次实际
79
79
  * 装了什么」;**环境变量值与 secret 不写进来**(所以 mcpServers 只记 name/command/args)。
80
80
  */
81
81
  export interface AgentSetupManifest {
@@ -322,7 +322,7 @@ export interface AgentContext {
322
322
  * 第三条反馈通道:上报本次运行的中性环境观测(如实际生效的 agent 配置、缓存命中状态)。
323
323
  * 落进 `AttemptRecord.facts` 成为一等观测量;不影响 Turn status 或 verdict,不参与
324
324
  * verdict / 评分 / 指纹。key 匹配 `[a-z0-9._-]{1,64}`,value 是标量;同 key 后写覆盖先写,
325
- * 非法 key 或非标量 value 抛错。形状与归属语义见 docs/feature/results/architecture.md#facts运行事实。
325
+ * 非法 key 或非标量 value 抛错。形状与归属语义见 docs/feature/record/architecture.md#facts运行事实。
326
326
  */
327
327
  fact(key: string, value: string | number | boolean): void;
328
328
  /**
@@ -270,7 +270,7 @@ describe("SessionManager · 多轮 Usage 累计", () => {
270
270
 
271
271
  // turn 级 usage 挂接:该轮 Turn.usage 经 onTurn 回报给 runner,由 runner 挂上 TimingNode
272
272
  // (src/runner/attempt.ts 的 onTurn → recorder.child({ usage })),show `--execution`/
273
- // `--timing` 的 turn 头行读 TimingNode.usage(见 docs/feature/results/architecture.md
273
+ // `--timing` 的 turn 头行读 TimingNode.usage(见 docs/feature/record/architecture.md
274
274
  // 「result.json」TimingNode.usage,src/show/render.ts 的 turnUsageText)。这里只测
275
275
  // session.ts 这一端:onTurn 收到的 usage 与该轮 Turn.usage 同值;没有 usage 的轮不传该字段。
276
276
  describe("SessionManager · onTurn 回报的 usage(turn 挂接 TimingNode 的数据来源)", () => {
@@ -438,7 +438,7 @@ export function lastAssistantText(events: readonly StreamEvent[]): string | unde
438
438
 
439
439
  /**
440
440
  * 每个字段只在某一轮真的带回该值时才累加,协议不提供就保持省略(见
441
- * docs/feature/results/architecture.md「Usage」:「每个字段只在协议真实提供该值时存在……
441
+ * docs/feature/record/architecture.md「Usage」:「每个字段只在协议真实提供该值时存在……
442
442
  * 不存在『默认 0』或『默认 1』的字段」)。此前 `requests` 用 `add.requests ?? 1` 累加,会让
443
443
  * 转录解析型 adapter(整个 attempt 只在末尾解析一次 transcript、天然不报每轮请求数)的一轮
444
444
  * send 被硬算成 1 个请求,一个内部发起了 21 次工具调用的 codex session 因此落盘
package/src/o11y/cost.ts CHANGED
@@ -57,7 +57,7 @@ function lookupBuiltin(model: string): Price | undefined {
57
57
 
58
58
  /**
59
59
  * 按 token 桶 × 单价估算一次运行的美元成本。逐桶相加成立的前提是 Usage 桶恒互斥
60
- * (inputTokens 不含 cache 命中,归一义务在 adapter,见 docs/feature/results/architecture.md#usage);
60
+ * (inputTokens 不含 cache 命中,归一义务在 adapter,见 docs/feature/record/architecture.md#usage);
61
61
  * cache 桶缺专门单价时退回 input 价——宁可高估,不静默低估。
62
62
  * 无 model / 查不到价(用户表 + 内置快照都没有)→ undefined。
63
63
  */
@@ -1,4 +1,4 @@
1
- // cases: docs/engineering/testing/unit/results.md
1
+ // cases: docs/engineering/testing/unit/record.md
2
2
  // 「Usage、facts 与失败命令证据落盘」桶恒互斥归一:bub tape 的 usage 是 Chat Completions
3
3
  // 形状,prompt_tokens_details.cached_tokens 是 prompt_tokens 子集,聚合前扣减;
4
4
  // 同对象的 cost 是实测计费,照常累进 costUSD。
@@ -237,7 +237,7 @@ export function parseBubTranscript(raw: string | undefined): ParsedTranscript {
237
237
  }
238
238
 
239
239
  // requests > 0 意味着至少一次 addUsage 真的读到了非零字段;整份 tape 没有任何 usage 时
240
- // input/output 也不该垫成 0(见 docs/feature/results/architecture.md#usage)。
240
+ // input/output 也不该垫成 0(见 docs/feature/record/architecture.md#usage)。
241
241
  const usage: Usage = requests > 0 ? { inputTokens, outputTokens } : {};
242
242
  if (cacheReadTokens > 0) usage.cacheReadTokens = cacheReadTokens;
243
243
  if (requests > 0) usage.requests = requests;
@@ -1,4 +1,4 @@
1
- // cases: docs/engineering/testing/unit/results.md
1
+ // cases: docs/engineering/testing/unit/record.md
2
2
  // 「Usage、facts 与失败命令证据落盘」桶恒互斥归一:codex transcript 的 cached_input_tokens
3
3
  // 是 input_tokens 子集,聚合前逐请求扣减。
4
4
  // bug: memory/estimatecost-openai-inclusive-cache-double-billed.md
@@ -419,7 +419,7 @@ export function parseCodexTranscript(raw: string | undefined): ParsedTranscript
419
419
  }
420
420
 
421
421
  // requests > 0 意味着至少一行真的带回了 usage;整份 transcript 没有任何 usage 行时
422
- // input/output 也不该垫成 0(见 docs/feature/results/architecture.md#usage)。
422
+ // input/output 也不该垫成 0(见 docs/feature/record/architecture.md#usage)。
423
423
  const usage: Usage = requests > 0 ? { inputTokens, outputTokens } : {};
424
424
  if (cacheReadTokens > 0) usage.cacheReadTokens = cacheReadTokens;
425
425
  if (reasoningTokens > 0) usage.reasoningTokens = reasoningTokens;
@@ -0,0 +1,198 @@
1
+ // Hermes sessions export / state.db 消息行解析器。方言只住这里,不进 core
2
+ // (契约见 docs/feature/adapters/sdk/hermes/README.md)。
3
+
4
+ import type { StreamEvent, Usage, ToolName, JsonValue } from "../../types.ts";
5
+ import type { ParsedTranscript } from "./index.ts";
6
+ import { GENERIC_VERB_ALIASES, normalizeToolName as normalizeShared } from "../tool-names.ts";
7
+
8
+ export const HERMES_TOOL_ALIASES: Record<string, ToolName> = {
9
+ ...GENERIC_VERB_ALIASES,
10
+ terminal: "shell",
11
+ read_file: "file_read",
12
+ write_file: "file_write",
13
+ edit_file: "file_edit",
14
+ web_extract: "web_fetch",
15
+ };
16
+
17
+ function normalizeToolName(name: string): ToolName {
18
+ return normalizeShared(name, HERMES_TOOL_ALIASES);
19
+ }
20
+
21
+ function get(obj: unknown, key: string): unknown {
22
+ return obj && typeof obj === "object" ? (obj as Record<string, unknown>)[key] : undefined;
23
+ }
24
+
25
+ function str(v: unknown): string | undefined {
26
+ return typeof v === "string" && v ? v : undefined;
27
+ }
28
+
29
+ function num(obj: unknown, ...keys: string[]): number {
30
+ for (const k of keys) {
31
+ const v = get(obj, k);
32
+ if (typeof v === "number" && Number.isFinite(v)) return v;
33
+ }
34
+ return 0;
35
+ }
36
+
37
+ function coerceArgs(value: unknown): JsonValue {
38
+ if (typeof value === "string") {
39
+ try {
40
+ return JSON.parse(value) as JsonValue;
41
+ } catch {
42
+ return value;
43
+ }
44
+ }
45
+ return (value ?? {}) as JsonValue;
46
+ }
47
+
48
+ function parseJsonField(value: unknown): unknown {
49
+ if (typeof value !== "string") return value;
50
+ try {
51
+ return JSON.parse(value);
52
+ } catch {
53
+ return value;
54
+ }
55
+ }
56
+
57
+ /**
58
+ * Hermes sessions export JSONL(一行一 session,或一行一 message)→ 标准事件 + usage。
59
+ */
60
+ export function parseHermesTranscript(raw: string | undefined): ParsedTranscript {
61
+ const events: StreamEvent[] = [];
62
+ let inputTokens = 0;
63
+ let outputTokens = 0;
64
+ let cacheReadTokens = 0;
65
+ let cacheCreationTokens = 0;
66
+ let costUSD = 0;
67
+ let requests = 0;
68
+ let parseSuccess = true;
69
+ let synth = 0;
70
+ const nextSynthId = (): string => `hm_${++synth}`;
71
+
72
+ if (!raw?.trim()) {
73
+ return { events, usage: {}, compactions: 0, parseSuccess: true };
74
+ }
75
+
76
+ const ingestSessionMeta = (obj: Record<string, unknown>): void => {
77
+ inputTokens += num(obj, "input_tokens", "inputTokens");
78
+ outputTokens += num(obj, "output_tokens", "outputTokens");
79
+ cacheReadTokens += num(obj, "cache_read_tokens", "cacheReadTokens");
80
+ cacheCreationTokens += num(obj, "cache_write_tokens", "cacheWriteTokens", "cacheCreationTokens");
81
+ const actual = num(obj, "actual_cost_usd", "actualCostUsd");
82
+ const estimated = num(obj, "estimated_cost_usd", "estimatedCostUsd");
83
+ costUSD += actual || estimated;
84
+ const apiCalls = num(obj, "api_call_count", "apiCallCount");
85
+ if (apiCalls) requests += apiCalls;
86
+ };
87
+
88
+ const ingestMessage = (msg: Record<string, unknown>): void => {
89
+ const role = str(msg.role) ?? "assistant";
90
+ const content = str(msg.content);
91
+
92
+ if (role === "tool" || str(msg.tool_call_id) || str(msg.toolCallId)) {
93
+ const callId = str(msg.tool_call_id) ?? str(msg.toolCallId) ?? nextSynthId();
94
+ const success = msg.is_error !== true && msg.isError !== true;
95
+ events.push({
96
+ type: "action.result",
97
+ callId,
98
+ output: (content ?? get(msg, "output") ?? null) as JsonValue,
99
+ status: success ? "completed" : "failed",
100
+ });
101
+ return;
102
+ }
103
+
104
+ const toolCallsRaw = parseJsonField(msg.tool_calls ?? msg.toolCalls);
105
+ if (Array.isArray(toolCallsRaw)) {
106
+ for (const call of toolCallsRaw) {
107
+ const fn = get(call, "function") as Record<string, unknown> | undefined;
108
+ const name =
109
+ str(get(fn, "name")) ?? str(get(call, "name")) ?? str(msg.tool_name) ?? "unknown";
110
+ const callId = str(get(call, "id")) ?? nextSynthId();
111
+ const args = coerceArgs(get(fn, "arguments") ?? get(call, "arguments") ?? get(call, "input"));
112
+ events.push({
113
+ type: "action.called",
114
+ callId,
115
+ name,
116
+ input: args,
117
+ tool: normalizeToolName(name),
118
+ });
119
+ }
120
+ }
121
+
122
+ if (content?.trim() && role !== "user" && role !== "system") {
123
+ events.push({ type: "message", role: "assistant", text: content });
124
+ } else if (content?.trim() && role === "user" && !Array.isArray(toolCallsRaw)) {
125
+ // 用户消息不进行为断言主路径;保留会污染 messageIncludes。跳过。
126
+ }
127
+
128
+ const reasoning = str(msg.reasoning) ?? str(msg.reasoning_content);
129
+ if (reasoning?.trim()) events.push({ type: "thinking", text: reasoning });
130
+ };
131
+
132
+ for (const line of raw.split("\n")) {
133
+ const trimmed = line.trim();
134
+ if (!trimmed) continue;
135
+ let obj: Record<string, unknown>;
136
+ try {
137
+ obj = JSON.parse(trimmed) as Record<string, unknown>;
138
+ } catch {
139
+ parseSuccess = false;
140
+ continue;
141
+ }
142
+
143
+ try {
144
+ // 整 session 导出:带 messages[]
145
+ if (Array.isArray(obj.messages)) {
146
+ ingestSessionMeta(obj);
147
+ for (const m of obj.messages) {
148
+ if (m && typeof m === "object") ingestMessage(m as Record<string, unknown>);
149
+ }
150
+ continue;
151
+ }
152
+ // 单行 message
153
+ if (str(obj.role)) {
154
+ ingestMessage(obj);
155
+ continue;
156
+ }
157
+ // sqlite dump 行:{session_id, role, content, tool_calls, ...}
158
+ if (str(obj.session_id) || str(obj.sessionId)) {
159
+ ingestMessage(obj);
160
+ continue;
161
+ }
162
+ ingestSessionMeta(obj);
163
+ } catch {
164
+ parseSuccess = false;
165
+ }
166
+ }
167
+
168
+ const usage: Usage = {};
169
+ if (inputTokens) usage.inputTokens = inputTokens;
170
+ if (outputTokens) usage.outputTokens = outputTokens;
171
+ if (cacheReadTokens) usage.cacheReadTokens = cacheReadTokens;
172
+ if (cacheCreationTokens) usage.cacheCreationTokens = cacheCreationTokens;
173
+ if (costUSD) usage.costUSD = costUSD;
174
+ if (requests) usage.requests = requests;
175
+
176
+ return { events, usage, compactions: 0, parseSuccess };
177
+ }
178
+
179
+ /** 从 export / stdout 文本里抠 session id。 */
180
+ export function sessionIdFromHermesOutput(raw: string | undefined): string | undefined {
181
+ if (!raw) return undefined;
182
+ for (const line of raw.split("\n")) {
183
+ const resume = line.match(/hermes\s+(?:chat\s+)?(?:--resume|-r)\s+(\S+)/i);
184
+ if (resume?.[1]) return resume[1].replace(/["'`]/g, "");
185
+ const labeled = line.match(/Session:\s*([A-Za-z0-9_-]+)/i);
186
+ if (labeled?.[1]) return labeled[1];
187
+ const trimmed = line.trim();
188
+ if (!trimmed.startsWith("{")) continue;
189
+ try {
190
+ const obj = JSON.parse(trimmed) as Record<string, unknown>;
191
+ const id = str(obj.id) ?? str(obj.session_id) ?? str(obj.sessionId);
192
+ if (id && (Array.isArray(obj.messages) || str(obj.source))) return id;
193
+ } catch {
194
+ // skip
195
+ }
196
+ }
197
+ return undefined;
198
+ }
@@ -22,6 +22,7 @@ export const OPENCLAW_TOOL_ALIASES: Record<string, ToolName> = {
22
22
  read: "file_read",
23
23
  write: "file_write",
24
24
  edit: "file_edit",
25
+ exec: "shell",
25
26
  process: "shell",
26
27
  browser: "web_fetch",
27
28
  };
@@ -203,7 +204,7 @@ export function parseOpenClawTranscript(raw: string | undefined): ParsedTranscri
203
204
  }
204
205
 
205
206
  // requests > 0 意味着至少一条消息真的带回了 usage;整份 transcript 没有任何 usage 时
206
- // input/output 也不该垫成 0(见 docs/feature/results/architecture.md#usage)。
207
+ // input/output 也不该垫成 0(见 docs/feature/record/architecture.md#usage)。
207
208
  const usage: Usage = requests > 0 ? { inputTokens, outputTokens } : {};
208
209
  if (cacheReadTokens > 0) usage.cacheReadTokens = cacheReadTokens;
209
210
  if (cacheCreationTokens > 0) usage.cacheCreationTokens = cacheCreationTokens;
@@ -224,6 +225,11 @@ export interface OpenClawRunJson {
224
225
  text?: string;
225
226
  /** 本次 run 的 session key(首轮 capture、后续 resume 用)。 */
226
227
  sessionId?: string;
228
+ /**
229
+ * 本轮 session transcript 绝对路径(`meta.agentMeta.sessionFile`)。
230
+ * 有的话优先按路径读,避开同目录 `*.trajectory.jsonl` 抢「最新」mtime。
231
+ */
232
+ sessionFile?: string;
227
233
  /** 封包报告的用量;没报就省略,不编造。 */
228
234
  usage?: Usage;
229
235
  /** 封包自报失败(error 字段 / status=error|failed / success=false)。 */
@@ -253,14 +259,29 @@ export function parseOpenClawRunJson(stdout: string | undefined): OpenClawRunJso
253
259
  str(get(doc, "reply")) ??
254
260
  str(get(doc, "message"));
255
261
 
262
+ const meta = get(doc, "meta");
263
+ const agentMeta = get(meta, "agentMeta");
264
+
256
265
  const sessionId =
257
266
  str(get(doc, "sessionId")) ??
258
267
  str(get(doc, "session_id")) ??
259
268
  str(get(doc, "sessionKey")) ??
260
269
  str(get(result, "sessionId")) ??
261
- str(get(result, "session_id"));
270
+ str(get(result, "session_id")) ??
271
+ str(get(agentMeta, "sessionId")) ??
272
+ str(get(agentMeta, "session_id"));
273
+
274
+ const sessionFile =
275
+ str(get(agentMeta, "sessionFile")) ??
276
+ str(get(doc, "sessionFile")) ??
277
+ str(get(result, "sessionFile"));
262
278
 
263
- const rawUsage = get(doc, "usage") ?? get(result, "usage") ?? get(get(doc, "meta"), "usage");
279
+ const rawUsage =
280
+ get(doc, "usage") ??
281
+ get(result, "usage") ??
282
+ get(meta, "usage") ??
283
+ get(agentMeta, "usage") ??
284
+ get(agentMeta, "lastCallUsage");
264
285
  let usage: Usage | undefined;
265
286
  if (rawUsage && typeof rawUsage === "object") {
266
287
  const input = num(rawUsage, "input", "input_tokens", "inputTokens", "prompt_tokens");
@@ -283,6 +304,7 @@ export function parseOpenClawRunJson(stdout: string | undefined): OpenClawRunJso
283
304
  return {
284
305
  ...(text !== undefined ? { text } : {}),
285
306
  ...(sessionId !== undefined ? { sessionId } : {}),
307
+ ...(sessionFile !== undefined ? { sessionFile } : {}),
286
308
  ...(usage !== undefined ? { usage } : {}),
287
309
  failed,
288
310
  };