niceeval 0.11.2 → 0.11.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/INDEX.md +1 -0
  2. package/README.md +4 -2
  3. package/README.zh.md +4 -2
  4. package/dist/agents/types.d.ts +2 -2
  5. package/dist/o11y/types.d.ts +3 -3
  6. package/dist/report/assets/colors.d.ts +1 -1
  7. package/dist/report/assets/colors.js +1 -1
  8. package/dist/report/components/attempt-detail/AttemptSource.js +1 -2
  9. package/dist/report/components/attempt-detail/compute.d.ts +4 -4
  10. package/dist/report/components/attempt-detail/compute.js +6 -6
  11. package/dist/report/components/attempt-detail/faces.d.ts +1 -1
  12. package/dist/report/components/attempt-detail/faces.js +3 -3
  13. package/dist/report/components/entity-lists/AttemptList.d.ts +3 -3
  14. package/dist/report/components/entity-lists/AttemptList.js +3 -3
  15. package/dist/report/components/entity-lists/ExperimentList.js +2 -2
  16. package/dist/report/components/entity-lists/compute.js +1 -1
  17. package/dist/report/components/entity-lists/faces.js +1 -1
  18. package/dist/report/components/entity-lists/index.d.ts +1 -1
  19. package/dist/report/components/entity-lists/index.js +1 -1
  20. package/dist/report/components/metric-views/chart-math.d.ts +2 -2
  21. package/dist/report/components/metric-views/chart-math.js +2 -2
  22. package/dist/report/components/metric-views/compute.d.ts +2 -3
  23. package/dist/report/components/metric-views/compute.js +3 -4
  24. package/dist/report/components/shared.d.ts +1 -1
  25. package/dist/report/components/site-components/compute.d.ts +5 -5
  26. package/dist/report/components/site-components/compute.js +6 -6
  27. package/dist/report/components/site-components/index.d.ts +7 -7
  28. package/dist/report/components/site-components/index.js +7 -7
  29. package/dist/report/components/site-components/scope-warnings.d.ts +1 -1
  30. package/dist/report/components/site-components/scope-warnings.js +3 -3
  31. package/dist/report/components/site-components/snapshot-diagnostics.js +1 -1
  32. package/dist/report/components/summaries/compute.d.ts +1 -1
  33. package/dist/report/components/summaries/compute.js +2 -2
  34. package/dist/report/components/summaries/faces.js +1 -1
  35. package/dist/report/components/summaries/index.d.ts +1 -1
  36. package/dist/report/components/summaries/index.js +1 -1
  37. package/dist/report/index.js +1 -1
  38. package/dist/report/model/aggregate.d.ts +2 -2
  39. package/dist/report/model/aggregate.js +2 -2
  40. package/dist/report/model/format.d.ts +2 -2
  41. package/dist/report/model/format.js +2 -2
  42. package/dist/report/model/metrics.js +3 -3
  43. package/dist/report/model/types.d.ts +11 -12
  44. package/dist/results/locator.js +1 -1
  45. package/dist/results/select.d.ts +4 -4
  46. package/dist/results/select.js +5 -5
  47. package/dist/results/types.d.ts +8 -8
  48. package/dist/runner/types.d.ts +14 -14
  49. package/dist/sandbox/resolve.d.ts +1 -1
  50. package/dist/sandbox/types.d.ts +1 -1
  51. package/docs-site/zh/reference/builtin-agents.mdx +21 -0
  52. package/docs-site/zh/reference/define-agent.mdx +1 -1
  53. package/docs-site/zh/reference/report-components.mdx +156 -74
  54. package/docs-site/zh/tutorials/custom-reports.mdx +112 -22
  55. package/docs-site/zh/tutorials/sandbox-agent.mdx +3 -4
  56. package/docs-site/zh/tutorials/sandbox-providers.mdx +15 -15
  57. package/docs-site/zh/tutorials/theming.mdx +169 -0
  58. package/package.json +2 -1
  59. package/src/agents/ai-sdk.test.ts +1 -1
  60. package/src/agents/bub-install-spec.ts +26 -8
  61. package/src/agents/bub.ts +40 -15
  62. package/src/agents/builtin.ts +11 -1
  63. package/src/agents/coding-cli-versions.ts +67 -0
  64. package/src/agents/hermes.ts +235 -0
  65. package/src/agents/index.ts +4 -0
  66. package/src/agents/langgraph.test.ts +1 -1
  67. package/src/agents/manifest.ts +1 -1
  68. package/src/agents/openai-compat.test.ts +1 -1
  69. package/src/agents/openclaw.ts +103 -24
  70. package/src/agents/opencode.ts +183 -0
  71. package/src/agents/sdk-streams.test.ts +1 -1
  72. package/src/agents/shared.ts +13 -2
  73. package/src/agents/types.ts +2 -2
  74. package/src/context/session.test.ts +1 -1
  75. package/src/context/session.ts +1 -1
  76. package/src/o11y/cost.ts +1 -1
  77. package/src/o11y/parsers/bub.test.ts +1 -1
  78. package/src/o11y/parsers/bub.ts +1 -1
  79. package/src/o11y/parsers/codex.test.ts +1 -1
  80. package/src/o11y/parsers/codex.ts +1 -1
  81. package/src/o11y/parsers/hermes.ts +198 -0
  82. package/src/o11y/parsers/openclaw.ts +25 -3
  83. package/src/o11y/parsers/opencode.ts +295 -0
  84. package/src/o11y/types.ts +3 -3
  85. package/src/report/assets/colors.ts +1 -1
  86. package/src/report/assets/styles.css +1 -1
  87. package/src/report/components/attempt-detail/AttemptAssertions.tsx +1 -1
  88. package/src/report/components/attempt-detail/AttemptConversation.tsx +1 -1
  89. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +1 -1
  90. package/src/report/components/attempt-detail/AttemptDiff.tsx +1 -1
  91. package/src/report/components/attempt-detail/AttemptError.tsx +1 -1
  92. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +1 -1
  93. package/src/report/components/attempt-detail/AttemptSource.tsx +2 -3
  94. package/src/report/components/attempt-detail/AttemptSummary.tsx +1 -1
  95. package/src/report/components/attempt-detail/AttemptTimeline.tsx +1 -1
  96. package/src/report/components/attempt-detail/AttemptTrace.tsx +1 -1
  97. package/src/report/components/attempt-detail/UsageTable.tsx +1 -1
  98. package/src/report/components/attempt-detail/attempt-components.test.tsx +1 -1
  99. package/src/report/components/attempt-detail/compute.ts +6 -6
  100. package/src/report/components/attempt-detail/faces.ts +3 -3
  101. package/src/report/components/attempt-detail/index.tsx +1 -1
  102. package/src/report/components/entity-lists/AttemptList.tsx +3 -3
  103. package/src/report/components/entity-lists/ExperimentList.tsx +2 -2
  104. package/src/report/components/entity-lists/compute.ts +1 -1
  105. package/src/report/components/entity-lists/faces.ts +1 -1
  106. package/src/report/components/entity-lists/index.tsx +1 -1
  107. package/src/report/components/metric-views/DeltaTable.tsx +1 -2
  108. package/src/report/components/metric-views/StabilityMatrix.tsx +1 -1
  109. package/src/report/components/metric-views/chart-math.test.ts +1 -1
  110. package/src/report/components/metric-views/chart-math.ts +2 -2
  111. package/src/report/components/metric-views/compute.ts +3 -4
  112. package/src/report/components/shared.ts +1 -1
  113. package/src/report/components/site-components/CopyFixPrompt.tsx +1 -1
  114. package/src/report/components/site-components/HeroCard.tsx +1 -1
  115. package/src/report/components/site-components/PoweredBy.tsx +1 -1
  116. package/src/report/components/site-components/ScopeWarnings.tsx +1 -1
  117. package/src/report/components/site-components/SnapshotDiagnostics.tsx +1 -1
  118. package/src/report/components/site-components/TraceWaterfall.tsx +1 -1
  119. package/src/report/components/site-components/compute.ts +6 -6
  120. package/src/report/components/site-components/index.tsx +7 -7
  121. package/src/report/components/site-components/scope-warnings.ts +3 -3
  122. package/src/report/components/site-components/snapshot-diagnostics.ts +1 -1
  123. package/src/report/components/summaries/ScopeSummary.tsx +2 -2
  124. package/src/report/components/summaries/compute.ts +2 -2
  125. package/src/report/components/summaries/faces.ts +1 -1
  126. package/src/report/components/summaries/index.tsx +1 -1
  127. package/src/report/index.ts +1 -1
  128. package/src/report/model/aggregate.ts +2 -2
  129. package/src/report/model/format.ts +2 -2
  130. package/src/report/model/metrics.ts +3 -3
  131. package/src/report/model/types.ts +12 -13
  132. package/src/results/annotated-source.test.ts +1 -1
  133. package/src/results/attempt-evidence.test.ts +1 -1
  134. package/src/results/copy.ts +3 -3
  135. package/src/results/format.ts +3 -3
  136. package/src/results/host-equivalence.test.ts +6 -6
  137. package/src/results/index.ts +1 -1
  138. package/src/results/locator.test.ts +2 -2
  139. package/src/results/locator.ts +1 -1
  140. package/src/results/open.ts +2 -2
  141. package/src/results/publish.ts +1 -1
  142. package/src/results/results.test.ts +6 -323
  143. package/src/results/select.test.ts +438 -0
  144. package/src/results/select.ts +6 -6
  145. package/src/results/skipped-notice.ts +1 -1
  146. package/src/results/truncate.ts +2 -2
  147. package/src/results/types.ts +9 -9
  148. package/src/results/writer.ts +4 -4
  149. package/src/runner/attempt.test.ts +2 -2
  150. package/src/runner/attempt.ts +6 -6
  151. package/src/runner/eval-source.test.ts +1 -1
  152. package/src/runner/eval-source.ts +1 -1
  153. package/src/runner/reporters/artifacts.ts +1 -1
  154. package/src/runner/run.ts +2 -2
  155. package/src/runner/timing.ts +1 -1
  156. package/src/runner/types.ts +14 -14
  157. package/src/sandbox/docker-agent-image.ts +36 -0
  158. package/src/sandbox/e2b-agent-template.test.ts +6 -21
  159. package/src/sandbox/e2b-agent-template.ts +44 -11
  160. package/src/sandbox/index.ts +8 -0
  161. package/src/sandbox/official-baselines.test.ts +175 -0
  162. package/src/sandbox/resolve.ts +1 -1
  163. package/src/sandbox/types.ts +1 -1
  164. package/src/scoring/diff.ts +1 -1
  165. package/src/scoring/types.ts +1 -1
  166. package/src/shared/facts.ts +1 -1
  167. package/src/show/index.ts +1 -1
  168. package/src/show/render.ts +1 -1
  169. package/src/show/show.test.ts +3 -3
  170. package/src/view/data.test.ts +2 -2
  171. package/src/view/data.ts +1 -1
  172. package/src/view/view-report.test.ts +1 -1
@@ -200,7 +200,7 @@ export interface ScriptResult {
200
200
  output: string;
201
201
  }
202
202
 
203
- // ── agent 归因增量(见 docs/feature/results/architecture.md「diff.json」与
203
+ // ── agent 归因增量(见 docs/feature/record/architecture.md「diff.json」与
204
204
  // docs/feature/sandbox/architecture.md「变更归因:send 窗口与分类账」)──
205
205
 
206
206
  /** diff.json 的落盘形状:按时序的窗口数组(逐窗口 delta 序列,不做跨窗口压缩)。 */
@@ -4,7 +4,7 @@
4
4
  // (attempt 级 facts 累加器 / experiment 级 facts 累加器),归属由调用方按当前作用域
5
5
  // 决定,不是本函数的职责;本函数只管「这次调用合不合法」与「写进哪个 Record」。
6
6
  // key 词法、value 标量、同 key 后写覆盖三条规则单源
7
- // docs/feature/results/architecture.md#facts运行事实。
7
+ // docs/feature/record/architecture.md#facts运行事实。
8
8
 
9
9
  /** fact() 接受的标量取值;与三处 ctx.fact() 签名、AttemptRecord.facts / SnapshotMeta.facts 同型。 */
10
10
  export type FactValue = string | number | boolean;
package/src/show/index.ts CHANGED
@@ -853,7 +853,7 @@ async function show(
853
853
  }
854
854
  // 无证据 flag:选中当前 report definition 里唯一的 attempt-input page,注入这份 evidence,
855
855
  // 走与其它 page 完全相同的 resolve → validate → render 管线(docs/feature/reports/show/attempt.md;
856
- // docs/feature/reports/library/attempt-detail.md「在 show 与 view 怎样渲染」)。不带 --report
856
+ // docs/feature/reports/components/attempt-detail/README.md「在 show 与 view 怎样渲染」)。不带 --report
857
857
  // 时装载内建 standard,其中就带这张 page;--report 指向的自定义报告没有声明 attempt-input page
858
858
  // 时报完整用户反馈,不回退到内建详情(三条解决路径都在错误文案里给出)。
859
859
  const report = await loadHostReport(cwd, flags.report);
@@ -514,7 +514,7 @@ interface CommandCard {
514
514
  timingNode?: TimingNode;
515
515
  }
516
516
 
517
- /** `commands.json` 的投影(docs/feature/results/architecture.md「commandsjson」);没有失败命令时 evidence.commands 为 null。 */
517
+ /** `commands.json` 的投影(docs/feature/record/architecture.md「commandsjson」);没有失败命令时 evidence.commands 为 null。 */
518
518
  function failedCommandsOf(evidence: AttemptEvidence): readonly FailedCommandEvidence[] {
519
519
  return evidence.commands ?? [];
520
520
  }
@@ -18,7 +18,7 @@
18
18
  // src/results/host-equivalence.test.ts 直接对 Selection 对象断言,不在本文件重复覆盖。
19
19
  //
20
20
  // fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
21
- // 依据是 docs/feature/results/architecture.md 的稳定磁盘契约,不经 writer 运行时 API(避免与并行重写的
21
+ // 依据是 docs/feature/record/architecture.md 的稳定磁盘契约,不经 writer 运行时 API(避免与并行重写的
22
22
  // niceeval/results 写入面签名耦合)。
23
23
 
24
24
  import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
@@ -52,7 +52,7 @@ afterAll(() => {
52
52
  setConfiguredLocale(undefined);
53
53
  });
54
54
 
55
- /** 一条 attempt 的最小 fixture;字段照 docs/feature/results/architecture.md 的 AttemptRecord。 */
55
+ /** 一条 attempt 的最小 fixture;字段照 docs/feature/record/architecture.md 的 AttemptRecord。 */
56
56
  type AttemptFixture = Pick<EvalResult, "id" | "verdict"> &
57
57
  Partial<
58
58
  Pick<
@@ -65,7 +65,7 @@ function res(id: string, verdict: Verdict, extra: Partial<AttemptFixture> = {}):
65
65
  return { id, verdict, attempt: 0, durationMs: 1000, assertions: [], ...extra };
66
66
  }
67
67
 
68
- /** 实验目录名的清洗:与 docs/feature/results/architecture.md 一致(/ 与非 [\w.@-] 换成 _)。 */
68
+ /** 实验目录名的清洗:与 docs/feature/record/architecture.md 一致(/ 与非 [\w.@-] 换成 _)。 */
69
69
  function cleanDirName(id: string): string {
70
70
  return id.replace(/[^\w.@-]/g, "_");
71
71
  }
@@ -10,7 +10,7 @@
10
10
  // docs/engineering/testing/e2e/report.md 对真实产物验收,不在本层断言。
11
11
  //
12
12
  // fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
13
- // 依据是 docs/feature/results/architecture.md 的稳定磁盘契约,不经 writer 运行时 API。
13
+ // 依据是 docs/feature/record/architecture.md 的稳定磁盘契约,不经 writer 运行时 API。
14
14
 
15
15
  import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
16
16
  import { tmpdir } from "node:os";
@@ -46,7 +46,7 @@ function res(id: string, verdict: Verdict, extra: Partial<AttemptFixture> = {}):
46
46
  return { id, verdict, attempt: 0, durationMs: 1000, assertions: [], ...extra };
47
47
  }
48
48
 
49
- /** 实验目录名的清洗:与 docs/feature/results/architecture.md 一致(/ 与非 [\w.@-] 换成 _)。 */
49
+ /** 实验目录名的清洗:与 docs/feature/record/architecture.md 一致(/ 与非 [\w.@-] 换成 _)。 */
50
50
  function cleanDirName(id: string): string {
51
51
  return id.replace(/[^\w.@-]/g, "_");
52
52
  }
package/src/view/data.ts CHANGED
@@ -115,7 +115,7 @@ export function incompatibleViewCommand(run: IncompatibleRun): string | undefine
115
115
  export function incompatibleHint(run: IncompatibleRun): string {
116
116
  const command = incompatibleViewCommand(run);
117
117
  if (command === undefined) {
118
- // 第三方 harness:如实报名字和版本,不拼 npx(docs/feature/results/library.md 的裁决)。
118
+ // 第三方 harness:如实报名字和版本,不拼 npx(docs/feature/record/library.md 的裁决)。
119
119
  return t("cli.view.incompatibleForeign", {
120
120
  dir: run.dir,
121
121
  name: run.producer?.name ?? "?",
@@ -17,7 +17,7 @@
17
17
  // 归 docs/engineering/testing/e2e/report.md §4/§5 对真实产物验收,不在本层断言。
18
18
  //
19
19
  // fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
20
- // 依据是 docs/feature/results/architecture.md 的稳定磁盘契约,不经 writer 运行时 API。
20
+ // 依据是 docs/feature/record/architecture.md 的稳定磁盘契约,不经 writer 运行时 API。
21
21
 
22
22
  import { mkdir, mkdtemp, rm, utimes, writeFile } from "node:fs/promises";
23
23
  import { tmpdir } from "node:os";