niceeval 0.11.2 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +1 -0
- package/README.md +4 -2
- package/README.zh.md +4 -2
- package/dist/agents/types.d.ts +2 -2
- package/dist/o11y/types.d.ts +3 -3
- package/dist/report/assets/colors.d.ts +1 -1
- package/dist/report/assets/colors.js +1 -1
- package/dist/report/components/attempt-detail/AttemptSource.js +1 -2
- package/dist/report/components/attempt-detail/compute.d.ts +4 -4
- package/dist/report/components/attempt-detail/compute.js +6 -6
- package/dist/report/components/attempt-detail/faces.d.ts +1 -1
- package/dist/report/components/attempt-detail/faces.js +3 -3
- package/dist/report/components/entity-lists/AttemptList.d.ts +3 -3
- package/dist/report/components/entity-lists/AttemptList.js +3 -3
- package/dist/report/components/entity-lists/ExperimentList.js +2 -2
- package/dist/report/components/entity-lists/compute.js +1 -1
- package/dist/report/components/entity-lists/faces.js +1 -1
- package/dist/report/components/entity-lists/index.d.ts +1 -1
- package/dist/report/components/entity-lists/index.js +1 -1
- package/dist/report/components/metric-views/chart-math.d.ts +2 -2
- package/dist/report/components/metric-views/chart-math.js +2 -2
- package/dist/report/components/metric-views/compute.d.ts +2 -3
- package/dist/report/components/metric-views/compute.js +3 -4
- package/dist/report/components/shared.d.ts +1 -1
- package/dist/report/components/site-components/compute.d.ts +5 -5
- package/dist/report/components/site-components/compute.js +6 -6
- package/dist/report/components/site-components/index.d.ts +7 -7
- package/dist/report/components/site-components/index.js +7 -7
- package/dist/report/components/site-components/scope-warnings.d.ts +1 -1
- package/dist/report/components/site-components/scope-warnings.js +3 -3
- package/dist/report/components/site-components/snapshot-diagnostics.js +1 -1
- package/dist/report/components/summaries/compute.d.ts +1 -1
- package/dist/report/components/summaries/compute.js +2 -2
- package/dist/report/components/summaries/faces.js +1 -1
- package/dist/report/components/summaries/index.d.ts +1 -1
- package/dist/report/components/summaries/index.js +1 -1
- package/dist/report/index.js +1 -1
- package/dist/report/model/aggregate.d.ts +2 -2
- package/dist/report/model/aggregate.js +2 -2
- package/dist/report/model/format.d.ts +2 -2
- package/dist/report/model/format.js +2 -2
- package/dist/report/model/metrics.js +3 -3
- package/dist/report/model/types.d.ts +11 -12
- package/dist/results/locator.js +1 -1
- package/dist/results/select.d.ts +4 -4
- package/dist/results/select.js +5 -5
- package/dist/results/types.d.ts +8 -8
- package/dist/runner/types.d.ts +14 -14
- package/dist/sandbox/resolve.d.ts +1 -1
- package/dist/sandbox/types.d.ts +1 -1
- package/docs-site/zh/reference/builtin-agents.mdx +21 -0
- package/docs-site/zh/reference/define-agent.mdx +1 -1
- package/docs-site/zh/reference/report-components.mdx +156 -74
- package/docs-site/zh/tutorials/custom-reports.mdx +112 -22
- package/docs-site/zh/tutorials/sandbox-agent.mdx +3 -4
- package/docs-site/zh/tutorials/sandbox-providers.mdx +15 -15
- package/docs-site/zh/tutorials/theming.mdx +169 -0
- package/package.json +2 -1
- package/src/agents/ai-sdk.test.ts +1 -1
- package/src/agents/bub-install-spec.ts +26 -8
- package/src/agents/bub.ts +40 -15
- package/src/agents/builtin.ts +11 -1
- package/src/agents/coding-cli-versions.ts +67 -0
- package/src/agents/hermes.ts +235 -0
- package/src/agents/index.ts +4 -0
- package/src/agents/langgraph.test.ts +1 -1
- package/src/agents/manifest.ts +1 -1
- package/src/agents/openai-compat.test.ts +1 -1
- package/src/agents/openclaw.ts +103 -24
- package/src/agents/opencode.ts +183 -0
- package/src/agents/sdk-streams.test.ts +1 -1
- package/src/agents/shared.ts +13 -2
- package/src/agents/types.ts +2 -2
- package/src/context/session.test.ts +1 -1
- package/src/context/session.ts +1 -1
- package/src/o11y/cost.ts +1 -1
- package/src/o11y/parsers/bub.test.ts +1 -1
- package/src/o11y/parsers/bub.ts +1 -1
- package/src/o11y/parsers/codex.test.ts +1 -1
- package/src/o11y/parsers/codex.ts +1 -1
- package/src/o11y/parsers/hermes.ts +198 -0
- package/src/o11y/parsers/openclaw.ts +25 -3
- package/src/o11y/parsers/opencode.ts +295 -0
- package/src/o11y/types.ts +3 -3
- package/src/report/assets/colors.ts +1 -1
- package/src/report/assets/styles.css +1 -1
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptConversation.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptDiff.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptError.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptSource.tsx +2 -3
- package/src/report/components/attempt-detail/AttemptSummary.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +1 -1
- package/src/report/components/attempt-detail/AttemptTrace.tsx +1 -1
- package/src/report/components/attempt-detail/UsageTable.tsx +1 -1
- package/src/report/components/attempt-detail/attempt-components.test.tsx +1 -1
- package/src/report/components/attempt-detail/compute.ts +6 -6
- package/src/report/components/attempt-detail/faces.ts +3 -3
- package/src/report/components/attempt-detail/index.tsx +1 -1
- package/src/report/components/entity-lists/AttemptList.tsx +3 -3
- package/src/report/components/entity-lists/ExperimentList.tsx +2 -2
- package/src/report/components/entity-lists/compute.ts +1 -1
- package/src/report/components/entity-lists/faces.ts +1 -1
- package/src/report/components/entity-lists/index.tsx +1 -1
- package/src/report/components/metric-views/DeltaTable.tsx +1 -2
- package/src/report/components/metric-views/StabilityMatrix.tsx +1 -1
- package/src/report/components/metric-views/chart-math.test.ts +1 -1
- package/src/report/components/metric-views/chart-math.ts +2 -2
- package/src/report/components/metric-views/compute.ts +3 -4
- package/src/report/components/shared.ts +1 -1
- package/src/report/components/site-components/CopyFixPrompt.tsx +1 -1
- package/src/report/components/site-components/HeroCard.tsx +1 -1
- package/src/report/components/site-components/PoweredBy.tsx +1 -1
- package/src/report/components/site-components/ScopeWarnings.tsx +1 -1
- package/src/report/components/site-components/SnapshotDiagnostics.tsx +1 -1
- package/src/report/components/site-components/TraceWaterfall.tsx +1 -1
- package/src/report/components/site-components/compute.ts +6 -6
- package/src/report/components/site-components/index.tsx +7 -7
- package/src/report/components/site-components/scope-warnings.ts +3 -3
- package/src/report/components/site-components/snapshot-diagnostics.ts +1 -1
- package/src/report/components/summaries/ScopeSummary.tsx +2 -2
- package/src/report/components/summaries/compute.ts +2 -2
- package/src/report/components/summaries/faces.ts +1 -1
- package/src/report/components/summaries/index.tsx +1 -1
- package/src/report/index.ts +1 -1
- package/src/report/model/aggregate.ts +2 -2
- package/src/report/model/format.ts +2 -2
- package/src/report/model/metrics.ts +3 -3
- package/src/report/model/types.ts +12 -13
- package/src/results/annotated-source.test.ts +1 -1
- package/src/results/attempt-evidence.test.ts +1 -1
- package/src/results/copy.ts +3 -3
- package/src/results/format.ts +3 -3
- package/src/results/host-equivalence.test.ts +6 -6
- package/src/results/index.ts +1 -1
- package/src/results/locator.test.ts +2 -2
- package/src/results/locator.ts +1 -1
- package/src/results/open.ts +2 -2
- package/src/results/publish.ts +1 -1
- package/src/results/results.test.ts +6 -323
- package/src/results/select.test.ts +438 -0
- package/src/results/select.ts +6 -6
- package/src/results/skipped-notice.ts +1 -1
- package/src/results/truncate.ts +2 -2
- package/src/results/types.ts +9 -9
- package/src/results/writer.ts +4 -4
- package/src/runner/attempt.test.ts +2 -2
- package/src/runner/attempt.ts +6 -6
- package/src/runner/eval-source.test.ts +1 -1
- package/src/runner/eval-source.ts +1 -1
- package/src/runner/reporters/artifacts.ts +1 -1
- package/src/runner/run.ts +2 -2
- package/src/runner/timing.ts +1 -1
- package/src/runner/types.ts +14 -14
- package/src/sandbox/docker-agent-image.ts +36 -0
- package/src/sandbox/e2b-agent-template.test.ts +6 -21
- package/src/sandbox/e2b-agent-template.ts +44 -11
- package/src/sandbox/index.ts +8 -0
- package/src/sandbox/official-baselines.test.ts +175 -0
- package/src/sandbox/resolve.ts +1 -1
- package/src/sandbox/types.ts +1 -1
- package/src/scoring/diff.ts +1 -1
- package/src/scoring/types.ts +1 -1
- package/src/shared/facts.ts +1 -1
- package/src/show/index.ts +1 -1
- package/src/show/render.ts +1 -1
- package/src/show/show.test.ts +3 -3
- package/src/view/data.test.ts +2 -2
- package/src/view/data.ts +1 -1
- package/src/view/view-report.test.ts +1 -1
package/src/scoring/types.ts
CHANGED
|
@@ -200,7 +200,7 @@ export interface ScriptResult {
|
|
|
200
200
|
output: string;
|
|
201
201
|
}
|
|
202
202
|
|
|
203
|
-
// ── agent 归因增量(见 docs/feature/
|
|
203
|
+
// ── agent 归因增量(见 docs/feature/record/architecture.md「diff.json」与
|
|
204
204
|
// docs/feature/sandbox/architecture.md「变更归因:send 窗口与分类账」)──
|
|
205
205
|
|
|
206
206
|
/** diff.json 的落盘形状:按时序的窗口数组(逐窗口 delta 序列,不做跨窗口压缩)。 */
|
package/src/shared/facts.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// (attempt 级 facts 累加器 / experiment 级 facts 累加器),归属由调用方按当前作用域
|
|
5
5
|
// 决定,不是本函数的职责;本函数只管「这次调用合不合法」与「写进哪个 Record」。
|
|
6
6
|
// key 词法、value 标量、同 key 后写覆盖三条规则单源
|
|
7
|
-
// docs/feature/
|
|
7
|
+
// docs/feature/record/architecture.md#facts运行事实。
|
|
8
8
|
|
|
9
9
|
/** fact() 接受的标量取值;与三处 ctx.fact() 签名、AttemptRecord.facts / SnapshotMeta.facts 同型。 */
|
|
10
10
|
export type FactValue = string | number | boolean;
|
package/src/show/index.ts
CHANGED
|
@@ -853,7 +853,7 @@ async function show(
|
|
|
853
853
|
}
|
|
854
854
|
// 无证据 flag:选中当前 report definition 里唯一的 attempt-input page,注入这份 evidence,
|
|
855
855
|
// 走与其它 page 完全相同的 resolve → validate → render 管线(docs/feature/reports/show/attempt.md;
|
|
856
|
-
// docs/feature/reports/
|
|
856
|
+
// docs/feature/reports/components/attempt-detail/README.md「在 show 与 view 怎样渲染」)。不带 --report
|
|
857
857
|
// 时装载内建 standard,其中就带这张 page;--report 指向的自定义报告没有声明 attempt-input page
|
|
858
858
|
// 时报完整用户反馈,不回退到内建详情(三条解决路径都在错误文案里给出)。
|
|
859
859
|
const report = await loadHostReport(cwd, flags.report);
|
package/src/show/render.ts
CHANGED
|
@@ -514,7 +514,7 @@ interface CommandCard {
|
|
|
514
514
|
timingNode?: TimingNode;
|
|
515
515
|
}
|
|
516
516
|
|
|
517
|
-
/** `commands.json` 的投影(docs/feature/
|
|
517
|
+
/** `commands.json` 的投影(docs/feature/record/architecture.md「commandsjson」);没有失败命令时 evidence.commands 为 null。 */
|
|
518
518
|
function failedCommandsOf(evidence: AttemptEvidence): readonly FailedCommandEvidence[] {
|
|
519
519
|
return evidence.commands ?? [];
|
|
520
520
|
}
|
package/src/show/show.test.ts
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
// src/results/host-equivalence.test.ts 直接对 Selection 对象断言,不在本文件重复覆盖。
|
|
19
19
|
//
|
|
20
20
|
// fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
|
|
21
|
-
// 依据是 docs/feature/
|
|
21
|
+
// 依据是 docs/feature/record/architecture.md 的稳定磁盘契约,不经 writer 运行时 API(避免与并行重写的
|
|
22
22
|
// niceeval/results 写入面签名耦合)。
|
|
23
23
|
|
|
24
24
|
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
|
@@ -52,7 +52,7 @@ afterAll(() => {
|
|
|
52
52
|
setConfiguredLocale(undefined);
|
|
53
53
|
});
|
|
54
54
|
|
|
55
|
-
/** 一条 attempt 的最小 fixture;字段照 docs/feature/
|
|
55
|
+
/** 一条 attempt 的最小 fixture;字段照 docs/feature/record/architecture.md 的 AttemptRecord。 */
|
|
56
56
|
type AttemptFixture = Pick<EvalResult, "id" | "verdict"> &
|
|
57
57
|
Partial<
|
|
58
58
|
Pick<
|
|
@@ -65,7 +65,7 @@ function res(id: string, verdict: Verdict, extra: Partial<AttemptFixture> = {}):
|
|
|
65
65
|
return { id, verdict, attempt: 0, durationMs: 1000, assertions: [], ...extra };
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
/** 实验目录名的清洗:与 docs/feature/
|
|
68
|
+
/** 实验目录名的清洗:与 docs/feature/record/architecture.md 一致(/ 与非 [\w.@-] 换成 _)。 */
|
|
69
69
|
function cleanDirName(id: string): string {
|
|
70
70
|
return id.replace(/[^\w.@-]/g, "_");
|
|
71
71
|
}
|
package/src/view/data.test.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
// docs/engineering/testing/e2e/report.md 对真实产物验收,不在本层断言。
|
|
11
11
|
//
|
|
12
12
|
// fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
|
|
13
|
-
// 依据是 docs/feature/
|
|
13
|
+
// 依据是 docs/feature/record/architecture.md 的稳定磁盘契约,不经 writer 运行时 API。
|
|
14
14
|
|
|
15
15
|
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
16
16
|
import { tmpdir } from "node:os";
|
|
@@ -46,7 +46,7 @@ function res(id: string, verdict: Verdict, extra: Partial<AttemptFixture> = {}):
|
|
|
46
46
|
return { id, verdict, attempt: 0, durationMs: 1000, assertions: [], ...extra };
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
-
/** 实验目录名的清洗:与 docs/feature/
|
|
49
|
+
/** 实验目录名的清洗:与 docs/feature/record/architecture.md 一致(/ 与非 [\w.@-] 换成 _)。 */
|
|
50
50
|
function cleanDirName(id: string): string {
|
|
51
51
|
return id.replace(/[^\w.@-]/g, "_");
|
|
52
52
|
}
|
package/src/view/data.ts
CHANGED
|
@@ -115,7 +115,7 @@ export function incompatibleViewCommand(run: IncompatibleRun): string | undefine
|
|
|
115
115
|
export function incompatibleHint(run: IncompatibleRun): string {
|
|
116
116
|
const command = incompatibleViewCommand(run);
|
|
117
117
|
if (command === undefined) {
|
|
118
|
-
// 第三方 harness:如实报名字和版本,不拼 npx(docs/feature/
|
|
118
|
+
// 第三方 harness:如实报名字和版本,不拼 npx(docs/feature/record/library.md 的裁决)。
|
|
119
119
|
return t("cli.view.incompatibleForeign", {
|
|
120
120
|
dir: run.dir,
|
|
121
121
|
name: run.producer?.name ?? "?",
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
// 归 docs/engineering/testing/e2e/report.md §4/§5 对真实产物验收,不在本层断言。
|
|
18
18
|
//
|
|
19
19
|
// fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
|
|
20
|
-
// 依据是 docs/feature/
|
|
20
|
+
// 依据是 docs/feature/record/architecture.md 的稳定磁盘契约,不经 writer 运行时 API。
|
|
21
21
|
|
|
22
22
|
import { mkdir, mkdtemp, rm, utimes, writeFile } from "node:fs/promises";
|
|
23
23
|
import { tmpdir } from "node:os";
|