niceeval 0.8.0 → 0.9.0

package/src/cli.ts

@@ -17,7 +17,7 @@ import { planCarry } from "./runner/fingerprint.ts";
 import { failureDetailFromResult } from "./runner/feedback/failure.ts";
 import { stopAllSandboxes, liveSandboxCount } from "./sandbox/registry.ts";
 import { evalLevelStats } from "./shared/verdict.ts";
-import { sandboxRecommendedConcurrency } from "./sandbox/resolve.ts";
+import { prepareRunSandboxes, resolvedSandboxRecommendedConcurrency } from "./runner/sandbox-selection.ts";
 import { Json, JUnit } from "./runner/reporters/json.ts";
 import { Artifacts as ArtifactsReporter } from "./runner/reporters/artifacts.ts";
 import {
@@ -105,7 +105,6 @@ interface Flags {
   timing?: "summary" | "full";
   keepSandbox?: "failed" | "all";
   all: boolean;
-  allowSensitiveArtifacts: boolean;
   window?: string;
   sandboxPath?: string;
   leaveRunning: boolean;
@@ -156,8 +155,6 @@ const FLAG_OPTIONS = {
   out: { type: "string" },
   /** `view` 命令专用:指定本地服务器监听端口。 */
   port: { type: "string" },
-  /** `view --out` 专用:对非发布根(快照没有 publish:{redaction:"applied"} 标记)导出时的显式确认——静态站会原样携带未消毒的证据文件。 */
-  "allow-sensitive-artifacts": { type: "boolean" },
   // show 的证据切面 / 时间轴 / 报告装载(docs-site/zh/how-to/viewing-results.mdx)。
   // 证据切面只认 `@<locator>`(或收窄到单个 eval 的前缀)选出的那一个 attempt——不再有
   // 数字 `--attempt`,选哪个 attempt 由 locator 精确指名,不是「先选 eval 再挑第几次」。
@@ -181,7 +178,7 @@ const FLAG_OPTIONS = {
   snapshot: { type: "string" },
   /** `show` / `view` 命令专用:用文件默认导出的 `defineReport(...)` 替换两者共用的默认报告。 */
   report: { type: "string" },
-  /** `show` / `view` 命令专用:选择多页报告的页(页 id 见 `show --report` 的页索引);`view` 里定初始页。 */
+  /** `show` / `view` 命令专用:选择报告的初始页;`show` 渲染该页并在尾部附其余页索引,`view` 以它作初始路由。未命中的页 id 按用法错误退出并列出可用页 id。 */
   page: { type: "string" },
   /** 只打印本次会匹配到的 eval × 运行配置,不实际执行(按下面 `--output` 选中的 profile 给出预览)。 */
   dry: { type: "boolean" },
@@ -307,7 +304,6 @@ function parseArgs(argv: string[]): { command: string; positionals: string[]; fl
     timing: values.timing === true ? (timingMode ?? "summary") : undefined,
     keepSandbox: values["keep-sandbox"] === true ? (keepSandboxTier ?? "failed") : undefined,
     all: values.all === true,
-    allowSensitiveArtifacts: values["allow-sensitive-artifacts"] === true,
     window: values.window as string | undefined,
     sandboxPath: values.path as string | undefined,
     leaveRunning: values["leave-running"] === true,
@@ -472,7 +468,7 @@ function evalsFilterFromExperiment(
 ): (id: string) => boolean {
   const patternFilter = makeFilter(patterns);
   let expFilter: (id: string) => boolean = () => true;
-  if (Array.isArray(evals)) expFilter = (id) => evals.includes(id) || evals.some((e) => id.startsWith(e + "/"));
+  if (Array.isArray(evals)) expFilter = makeFilter(evals);
   else if (typeof evals === "function") expFilter = evals;
   return (id) => expFilter(id) && patternFilter(id);
 }
@@ -591,7 +587,7 @@ async function main(): Promise<void> {
       ...(flags.page !== undefined ? { page: flags.page } : {}),
     };
     if (flags.out) {
-      const out = await buildView({ input: viewInput.input, out: flags.out, allowSensitiveArtifacts: flags.allowSensitiveArtifacts, scan }).catch(exitOnViewUserError);
+      const out = await buildView({ input: viewInput.input, out: flags.out, scan }).catch(exitOnViewUserError);
       process.stdout.write(t("cli.view.exportedDir", { out }));
       process.exit(0);
     }
@@ -775,6 +771,9 @@ async function main(): Promise<void> {
     process.exit(1);
   }
 
+  // environments 查表在 dry-run 与真实运行共用的规划边界完成；缺表项在任何沙箱/agent 花费前穷举失败。
+  prepareRunSandboxes(evals, agentRuns, config.sandbox);
+
   if (flags.dry) {
     // --dry 只按所选 profile 打印计划,不运行、不落盘 —— 三种 profile 各自的展示逻辑
     // 都在 runner/feedback/{human,agent,ci}.ts 里,这里只负责拼数据、选函数、写流、退出。
@@ -830,7 +829,7 @@ async function main(): Promise<void> {
   // envelope 展示的"携入"就会和 run.ts 真实调度的"携入"对不上(见 memory 的
   // live-carry-row-shows-waiting-forever)。
   const priorResults = flags.force ? undefined : await loadLatestResultsPerEval(join(cwd, ".niceeval"));
-  const carryPlan = priorResults?.length ? await planCarry(evals, agentRuns, priorResults) : undefined;
+  const carryPlan = priorResults?.length ? await planCarry(evals, agentRuns, priorResults, config.sandbox) : undefined;
   const reusedFailures = (carryPlan?.carriedResults ?? [])
     .map(failureDetailFromResult)
     .filter((failure) => failure !== undefined);
@@ -838,8 +837,7 @@ async function main(): Promise<void> {
   // 无全局默认:并发上限由 sandbox provider 的推荐值决定(多个 agentRun 各有 sandbox 时取
   // 最小值,最保守的 provider 决定上限)。同一个值既进 RunFeedbackPlan.shape,也传给 runEvals——
   // 两处必须是同一个数字,dashboard 展示的并发上限不能和真实调度的并发上限对不上。
-  const sandboxRecs = agentRuns.map((r) => sandboxRecommendedConcurrency(r.sandbox));
-  const sandboxDefaultConcurrency = sandboxRecs.length > 0 ? Math.min(...sandboxRecs) : 10;
+  const sandboxDefaultConcurrency = resolvedSandboxRecommendedConcurrency(evals, agentRuns, config.sandbox);
   const maxConcurrency =
     flags.maxConcurrency ??
     envNumber("NICEEVAL_MAX_CONCURRENCY") ??

package/src/context/context.ts

@@ -165,7 +165,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
     return brief(value, 4000);
   }
 
-  /** 失败断言的 evidence:被检查值自带命令摘要(CommandResult.command)时就是「命令行本身」。 */
+  /** evidence 不区分 pass/fail(与 judge 同口径):被检查值自带命令摘要(CommandResult.command)时就是「命令行本身」。 */
   function checkedValueEvidence(value: unknown): string | undefined {
     const command = asCommandResult(value)?.command;
     return typeof command === "string" && command.length > 0 ? command : undefined;
@@ -378,12 +378,15 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
         evaluate: async (sc) => {
           const resolved = await resolveValue(value, sc);
           const score = await assertion.score(resolved);
-          if (computePassed(spec.severity, spec.threshold, score)) return score;
+          const evidence = checkedValueEvidence(resolved);
+          if (computePassed(spec.severity, spec.threshold, score)) {
+            return evidence !== undefined ? { score, evidence } : score;
+          }
           return {
             score,
             expected: assertion.expected,
             received: previewCheckedValue(resolved),
-            ...(checkedValueEvidence(resolved) !== undefined ? { evidence: checkedValueEvidence(resolved) } : {}),
+            ...(evidence !== undefined ? { evidence } : {}),
           };
         },
       };
@@ -395,18 +398,21 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
       const score = await assertion.score(v);
       // require 恒为硬门槛(不过即中止 eval),判定口径与 finalize 同一份 computePassed。
       const passed = computePassed("gate", assertion.threshold, score);
+      const evidence = checkedValueEvidence(v);
       collector.record({
         name: assertion.name,
         severity: "gate",
         threshold: assertion.threshold,
         evaluate: () =>
           passed
-            ? score
+            ? evidence !== undefined
+              ? { score, evidence }
+              : score
             : {
                 score,
                 expected: assertion.expected,
                 received: previewCheckedValue(v),
-                ...(checkedValueEvidence(v) !== undefined ? { evidence: checkedValueEvidence(v) } : {}),
+                ...(evidence !== undefined ? { evidence } : {}),
               },
       });
       if (!passed) throw new EvalRequirementFailed(assertion.name);

package/src/define.ts

@@ -55,6 +55,9 @@ export function defineEval(def: EvalDef): EvalDef {
   if (typeof def.test !== "function") {
     throw new Error(t("define.evalTestRequired"));
   }
+  if (def.environment !== undefined && def.environment.trim().length === 0) {
+    throw new Error(t("define.evalEnvironmentEmpty"));
+  }
   return def;
 }
 

package/src/i18n/en.ts

@@ -24,6 +24,8 @@ export const en = {
   "skill.repoAmbiguous": "Repo skill {{source}} contains multiple skills; select which ones to enable with `skills: [...]`. Available: {{available}}.",
   "skill.repoUnknownSkill": "Repo skill {{source}} (ref: {{ref}}) has no skill named \"{{skill}}\". Available: {{available}}.",
   "skill.copyFailed": "Could not install skill \"{{name}}\" into {{dest}}:\n{{tail}}",
+  "mcp.ambiguousTransport":
+    "MCP server \"{{name}}\" specifies both \"command\" and \"url\" — pick one transport: \"command\" for a local stdio server, \"url\" for a remote Streamable HTTP endpoint.",
   "plugin.marketplaceFailed": "Could not connect {{agent}} marketplace \"{{name}}\" (source: {{source}}, ref: {{ref}}):\n{{tail}}",
   "plugin.marketplaceVerifyFailed": "Could not read back the registered marketplace list after adding {{agent}} marketplace \"{{name}}\" ({{command}}):\n{{tail}}",
   "plugin.marketplaceNameMismatch":
@@ -83,8 +85,8 @@ export const en = {
     "      --diff[=file] sandbox workspace file-change summary; =file expands one file\n" +
     "      --history   per experiment × eval execution timeline (mutually exclusive with --report)\n" +
     "      --results <dir>   pin a results root    --experiment <id>   one experiment\n" +
-    "      --report <file>   custom report    --page <id>   pick a page (multi-page\n" +
-    "        reports print a page index with copyable commands)\n" +
+    "      --report <file>   custom report    --page <id>   pick the initial page (multi-page\n" +
+    "        reports render it, then list the rest as a page index with copyable commands)\n" +
     "  niceeval list                                       list discovered evals\n" +
     "  niceeval view [eval-id-prefix…] [--out dir] [--port n] [--no-open]\n" +
     "      report pages + evidence rooms; --report <file> swaps in your report\n" +
@@ -151,6 +153,7 @@ export const en = {
   "context.turnFailedDefault": "This send returned failed (turn status = failed)",
   "define.agentNameRequired": "defineAgent requires name.",
   "define.evalIdRejected": "defineEval does not accept id; ids are derived from file paths.",
+  "define.evalEnvironmentEmpty": "defineEval environment must be a non-empty profile id when provided.",
   "define.evalTestRequired": "defineEval requires an async test(t) function.",
   "define.experimentAgentRequired": "defineExperiment requires agent.",
   "define.experimentFlagNotJson": "experiment.flags.{{key}} is not JSON-serializable (functions / undefined / cycles / bigint are not allowed); flags are persisted verbatim into result snapshots and must be plain JSON.",

package/src/i18n/zh-CN.ts

@@ -22,6 +22,8 @@ export const zhCN = {
   "skill.repoAmbiguous": "repo skill {{source}} 里有多个 skill,必须用 `skills: [...]` 明确选择要启用哪些。可选:{{available}}。",
   "skill.repoUnknownSkill": "repo skill {{source}}(ref: {{ref}})里没有名为 \"{{skill}}\" 的 skill。可选:{{available}}。",
   "skill.copyFailed": "skill \"{{name}}\" 装进 {{dest}} 失败:\n{{tail}}",
+  "mcp.ambiguousTransport":
+    "MCP server \"{{name}}\" 同时给出了 command 和 url——二选一:本地 stdio 进程写 command,远程 Streamable HTTP 端点写 url。",
   "plugin.marketplaceFailed": "{{agent}} marketplace \"{{name}}\" 连接失败(source: {{source}}, ref: {{ref}}):\n{{tail}}",
   "plugin.marketplaceVerifyFailed": "{{agent}} marketplace \"{{name}}\" add 后回读注册列表失败({{command}}):\n{{tail}}",
   "plugin.marketplaceNameMismatch":
@@ -79,7 +81,8 @@ export const zhCN = {
     "      --diff[=文件] agent 归因的文件改动摘要;=文件 按窗口展开单个文件\n" +
     "      --history   逐 experiment × eval 的执行时间轴(与 --report 互斥)\n" +
     "      --results <目录> 钉死结果根   --experiment <id> 只看该实验\n" +
-    "      --report <文件> 自定义报告   --page <id> 选页(多页报告先输出页索引)\n" +
+    "      --report <文件> 自定义报告   --page <id> 定初始页(多页报告渲染该页,\n" +
+    "        尾部再附其余页索引)\n" +
     "  niceeval list                            列出发现到的 eval\n" +
     "  niceeval view [eval-id 前缀…] [--out 目录] [--port n] [--no-open]\n" +
     "      报告页 + 证据室;--report <文件> 整槽换成自定义报告(与 show 同一文件)\n" +
@@ -145,6 +148,7 @@ export const zhCN = {
   "context.turnFailedDefault": "本轮 send 返回 failed(turn status = failed)",
   "define.agentNameRequired": "defineAgent 需要 name。",
   "define.evalIdRejected": "defineEval 不接受 id —— id 由文件路径推导。",
+  "define.evalEnvironmentEmpty": "defineEval 的 environment 如有提供，必须是非空的 profile id。",
   "define.evalTestRequired": "defineEval 需要一个 async test(t) 函数。",
   "define.experimentAgentRequired": "defineExperiment 需要 agent。",
   "define.experimentFlagNotJson": "experiment.flags.{{key}} 不是可 JSON 序列化的值(函数 / undefined / 循环引用 / bigint 不允许);flags 会原样进入结果快照,必须是纯 JSON。",

package/src/index.ts

@@ -35,6 +35,7 @@ export type {
   RunShape,
   AssertionResult,
   ExperimentRunInfo,
+  SandboxRunInfo,
   DiffData,
   DiffView,
   ScriptResult,

package/src/report/built-in/index.tsx

@@ -1,9 +1,10 @@
-// niceeval/report/built-in —— 包里自带的一个报告文件,没有任何私有钩子
-// (docs/feature/reports/library/built-in.md)。裸 `niceeval show` 与 `niceeval view`
-// 装载的就是这份默认导出:一份普通 defineReport,与用户的 --report 文件同层、
-// 走同一条 装载 → resolve → validate → render 管线。「builtin」不是装载逻辑里的类别,
-// 只是宿主默认拿哪个值的事实。
+// niceeval/report/built-in —— 内建视图的家:每个内建视图是一份普通 defineReport 成品,
+// 有自己的名字、一个源文件,按名字具名导出;默认导出恒等于 standard——裸宿主装载的那份
+// (docs/feature/reports/library/built-in.md)。新增内建视图 = 新文件 + 新具名导出,
+// 不需要注册表,也不改变装载管线。
 
-import { ExperimentComparison, defineReport } from "../index.ts";
+import { standard } from "./standard.tsx";
 
-export default defineReport(<ExperimentComparison />);
+export { standard };
+
+export default standard;

package/src/report/built-in/standard.tsx

@@ -0,0 +1,59 @@
+// standard —— 内建视图之一:报告 / Attempts / 追踪三页的通用结果站,没有任何私有钩子
+// (docs/feature/reports/library/built-in.md)。裸 `niceeval show` 与 `niceeval view`
+// 装载的就是它:一份普通 defineReport,与用户的 --report 文件同层、走同一条
+// 装载 → resolve → validate → render 管线。「builtin」不是装载逻辑里的类别,
+// 只是宿主默认拿哪个值的事实;用户报告经 defineReport({ extends: standard, … }) 整站复用。
+//
+// 三页站点:report(Hero + 警告 + 批量修复 prompt + 实验对比)、attempts(带过滤的
+// attempt 全列表)、traces(执行瀑布)。裸宿主导航上能看到的一切内容都在这份定义里;
+// 宿主保留的只有机器(docs/feature/reports/architecture.md「宿主保留的只有机器」)。
+
+import {
+  AttemptList,
+  Col,
+  CopyFixPrompt,
+  ExperimentComparison,
+  Hero,
+  ScopeWarnings,
+  TraceWaterfall,
+  defineReport,
+} from "../index.ts";
+
+export const standard = defineReport({
+  pages: [
+    {
+      id: "report",
+      title: { en: "Report", "zh-CN": "报告" },
+      content: (
+        <Col>
+          <Hero />
+          <ScopeWarnings />
+          <CopyFixPrompt />
+          <ExperimentComparison />
+        </Col>
+      ),
+    },
+    {
+      id: "attempts",
+      title: "Attempts",
+      content: (
+        <Col>
+          <Hero />
+          <ScopeWarnings />
+          <AttemptList filter />
+        </Col>
+      ),
+    },
+    {
+      id: "traces",
+      title: { en: "Traces", "zh-CN": "追踪" },
+      content: (
+        <Col>
+          <Hero />
+          <ScopeWarnings />
+          <TraceWaterfall />
+        </Col>
+      ),
+    },
+  ],
+});