niceeval 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +77 -45
- package/dist/agents/types.d.ts +28 -6
- package/dist/i18n/en.d.ts +2 -0
- package/dist/i18n/zh-CN.d.ts +2 -0
- package/dist/report/built-in/index.d.ts +3 -2
- package/dist/report/built-in/index.js +7 -8
- package/dist/report/built-in/standard.d.ts +1 -0
- package/dist/report/built-in/standard.js +30 -0
- package/dist/report/components.d.ts +72 -6
- package/dist/report/components.js +159 -10
- package/dist/report/compute.d.ts +31 -4
- package/dist/report/compute.js +131 -12
- package/dist/report/index.d.ts +4 -4
- package/dist/report/index.js +3 -2
- package/dist/report/locale.d.ts +39 -1
- package/dist/report/locale.js +69 -0
- package/dist/report/react/AttemptList.d.ts +3 -1
- package/dist/report/react/AttemptList.js +3 -3
- package/dist/report/react/CopyFixPrompt.d.ts +12 -0
- package/dist/report/react/CopyFixPrompt.js +12 -0
- package/dist/report/react/HeroCard.d.ts +13 -0
- package/dist/report/react/HeroCard.js +35 -0
- package/dist/report/react/PoweredBy.d.ts +5 -0
- package/dist/report/react/PoweredBy.js +7 -0
- package/dist/report/react/ScopeWarnings.d.ts +12 -0
- package/dist/report/react/ScopeWarnings.js +18 -0
- package/dist/report/react/TraceWaterfall.d.ts +14 -0
- package/dist/report/react/TraceWaterfall.js +22 -0
- package/dist/report/react/index.d.ts +6 -1
- package/dist/report/react/index.js +6 -0
- package/dist/report/report.d.ts +22 -6
- package/dist/report/report.js +66 -53
- package/dist/report/scope-warnings.d.ts +28 -0
- package/dist/report/scope-warnings.js +101 -0
- package/dist/report/text/faces.d.ts +19 -1
- package/dist/report/text/faces.js +61 -0
- package/dist/report/tree.js +6 -1
- package/dist/report/types.d.ts +45 -10
- package/dist/report/web.d.ts +5 -4
- package/dist/report/web.js +7 -22
- package/dist/results/select.d.ts +15 -2
- package/dist/results/select.js +75 -10
- package/dist/results/types.d.ts +26 -11
- package/dist/runner/fingerprint.d.ts +3 -3
- package/dist/runner/sandbox-selection.d.ts +12 -0
- package/dist/runner/types.d.ts +18 -6
- package/dist/sandbox/types.d.ts +12 -0
- package/docs-site/zh/explanation/evals.mdx +2 -1
- package/docs-site/zh/explanation/experiment.mdx +2 -0
- package/docs-site/zh/how-to/custom-reports.mdx +22 -6
- package/docs-site/zh/how-to/publish-report.mdx +6 -8
- package/docs-site/zh/how-to/viewing-results.mdx +3 -3
- package/docs-site/zh/how-to/write-experiment.mdx +40 -1
- package/docs-site/zh/reference/builtin-agents.mdx +40 -3
- package/docs-site/zh/reference/cli.mdx +1 -2
- package/docs-site/zh/reference/define-eval.mdx +8 -0
- package/docs-site/zh/reference/official-adapters.mdx +10 -5
- package/docs-site/zh/reference/report-components.mdx +2 -2
- package/docs-site/zh/reference/results-data.mdx +2 -4
- package/package.json +2 -1
- package/src/agents/bub.ts +13 -1
- package/src/agents/claude-code.test.ts +43 -1
- package/src/agents/claude-code.ts +32 -14
- package/src/agents/codex.test.ts +168 -1
- package/src/agents/codex.ts +51 -15
- package/src/agents/mcp.ts +31 -0
- package/src/agents/post-setup.ts +33 -0
- package/src/agents/types.ts +28 -7
- package/src/cli.ts +9 -11
- package/src/context/context.ts +11 -5
- package/src/define.ts +3 -0
- package/src/i18n/en.ts +5 -2
- package/src/i18n/zh-CN.ts +5 -1
- package/src/index.ts +1 -0
- package/src/report/built-in/index.tsx +8 -7
- package/src/report/built-in/standard.tsx +59 -0
- package/src/report/components.tsx +231 -17
- package/src/report/compute.ts +146 -21
- package/src/report/dual-render.test.tsx +139 -12
- package/src/report/index.ts +20 -1
- package/src/report/locale.ts +83 -1
- package/src/report/react/AttemptList.tsx +13 -1
- package/src/report/react/CopyFixPrompt.tsx +37 -0
- package/src/report/react/HeroCard.tsx +59 -0
- package/src/report/react/PoweredBy.tsx +20 -0
- package/src/report/react/ScopeWarnings.tsx +74 -0
- package/src/report/react/TraceWaterfall.tsx +78 -0
- package/src/report/react/enhance.js +14 -0
- package/src/report/react/index.tsx +11 -0
- package/src/report/react/styles.css +187 -7
- package/src/report/report.test.ts +2 -20
- package/src/report/report.ts +97 -62
- package/src/report/scope-warnings.ts +155 -0
- package/src/report/site-components.test.tsx +526 -0
- package/src/report/text/faces.ts +66 -0
- package/src/report/tree.ts +8 -1
- package/src/report/types.ts +51 -11
- package/src/report/web.ts +7 -40
- package/src/results/copy.ts +15 -78
- package/src/results/host-equivalence.test.ts +5 -1
- package/src/results/open.ts +5 -4
- package/src/results/publish.ts +4 -146
- package/src/results/results.test.ts +86 -9
- package/src/results/select.ts +78 -10
- package/src/results/types.ts +27 -7
- package/src/runner/attempt.ts +10 -9
- package/src/runner/discover.test.ts +9 -1
- package/src/runner/discover.ts +3 -3
- package/src/runner/fingerprint.ts +9 -4
- package/src/runner/ledger.test.ts +30 -1
- package/src/runner/ledger.ts +26 -4
- package/src/runner/run.ts +5 -1
- package/src/runner/sandbox-selection.test.ts +131 -0
- package/src/runner/sandbox-selection.ts +110 -0
- package/src/runner/types.ts +19 -2
- package/src/sandbox/types.ts +6 -0
- package/src/show/index.ts +17 -10
- package/src/show/render.ts +11 -11
- package/src/show/report-host.test.ts +32 -15
- package/src/show/report-host.ts +5 -4
- package/src/show/show.test.ts +140 -3
- package/src/view/app/App.test.tsx +78 -17
- package/src/view/app/App.tsx +17 -78
- package/src/view/app/components/CopyControls.tsx +4 -42
- package/src/view/app/i18n.ts +5 -227
- package/src/view/app/lib/rows.ts +3 -21
- package/src/view/app/shared.ts +1 -3
- package/src/view/app/types.ts +2 -2
- package/src/view/artifact-serving.test.ts +1 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +20 -20
- package/src/view/data.ts +2 -13
- package/src/view/index.ts +1 -12
- package/src/view/server.ts +0 -2
- package/src/view/shared/types.ts +11 -6
- package/src/view/site-parity.test.ts +1 -1
- package/src/view/site.ts +1 -1
- package/src/view/styles.css +6 -266
- package/src/view/view-report.test.ts +64 -27
- package/src/view/app/components/LazyArtifact.tsx +0 -51
- package/src/view/app/components/SkippedRunsBanner.tsx +0 -140
- package/src/view/app/pages/AttemptsPage.tsx +0 -80
- package/src/view/app/pages/TracesPage.tsx +0 -35
|
@@ -218,7 +218,7 @@ describe("claudeCodeAgent settingsFile · setup", () => {
|
|
|
218
218
|
await rm(root, { recursive: true, force: true });
|
|
219
219
|
});
|
|
220
220
|
|
|
221
|
-
const ctx = {} as AgentContext; //
|
|
221
|
+
const ctx = {} as AgentContext; // 本组用例不配 postSetup,setup 不会读 ctx 的字段
|
|
222
222
|
|
|
223
223
|
it("原始字节原样上传并 mv 成用户级 ~/.claude/settings.json;manifest 记项目相对路径 + SHA-256,不落正文", async () => {
|
|
224
224
|
const body = '{\n "$schema": "https://json.schemastore.org/claude-code-settings.json",\n "permissions": { "deny": ["WebSearch", "WebFetch"] }\n}\n';
|
|
@@ -271,3 +271,45 @@ describe("claudeCodeAgent settingsFile · setup", () => {
|
|
|
271
271
|
expect(box.written["__niceeval__/agent-setup.json"]).toBeUndefined();
|
|
272
272
|
});
|
|
273
273
|
});
|
|
274
|
+
|
|
275
|
+
describe("claudeCodeAgent mcpServers · 形态落位", () => {
|
|
276
|
+
const ctx = {} as AgentContext;
|
|
277
|
+
|
|
278
|
+
it("HTTP 形态写成 ~/.claude.json 的 type http + url + headers 条目,stdio 条目不变;manifest 只记非 secret 字段", async () => {
|
|
279
|
+
const box = sb();
|
|
280
|
+
await claudeCodeAgent({
|
|
281
|
+
apiKey: "k",
|
|
282
|
+
mcpServers: [
|
|
283
|
+
{ name: "browser", command: "npx", args: ["-y", "server"], env: { TOKEN: "env-sekret" } },
|
|
284
|
+
{ name: "team-memory", url: "https://mem.example.com/mcp/", headers: { Authorization: "Bearer sekret" } },
|
|
285
|
+
],
|
|
286
|
+
}).setup!(asSandbox(box), ctx);
|
|
287
|
+
|
|
288
|
+
// 用户级 MCP 配置经 heredoc 写进 ~/.claude.json(shared.writeFile),内容在命令里。
|
|
289
|
+
const write = box.commands.find((c) => c.includes("cat > ~/.claude.json"))!;
|
|
290
|
+
expect(write).toContain('"type": "http"');
|
|
291
|
+
expect(write).toContain('"url": "https://mem.example.com/mcp/"');
|
|
292
|
+
expect(write).toContain('"Authorization": "Bearer sekret"');
|
|
293
|
+
expect(write).toContain('"command": "npx"');
|
|
294
|
+
|
|
295
|
+
const manifestRaw = box.written["__niceeval__/agent-setup.json"]!;
|
|
296
|
+
const manifest = JSON.parse(manifestRaw) as AgentSetupManifest;
|
|
297
|
+
expect(manifest.mcpServers).toEqual([
|
|
298
|
+
{ name: "browser", command: "npx", args: ["-y", "server"] },
|
|
299
|
+
{ name: "team-memory", url: "https://mem.example.com/mcp/" },
|
|
300
|
+
]);
|
|
301
|
+
expect(manifestRaw).not.toContain("sekret");
|
|
302
|
+
});
|
|
303
|
+
|
|
304
|
+
it("边界:HTTP 形态无 headers → 条目不带 headers 字段", async () => {
|
|
305
|
+
const box = sb();
|
|
306
|
+
await claudeCodeAgent({
|
|
307
|
+
apiKey: "k",
|
|
308
|
+
mcpServers: [{ name: "team-memory", url: "https://mem.example.com/mcp/" }],
|
|
309
|
+
}).setup!(asSandbox(box), ctx);
|
|
310
|
+
|
|
311
|
+
const write = box.commands.find((c) => c.includes("cat > ~/.claude.json"))!;
|
|
312
|
+
expect(write).toContain('"type": "http"');
|
|
313
|
+
expect(write).not.toContain("headers");
|
|
314
|
+
});
|
|
315
|
+
});
|
|
@@ -14,7 +14,9 @@ import {
|
|
|
14
14
|
import { mapClaudeCodeSpans } from "../o11y/otlp/mappers/claude-code.ts";
|
|
15
15
|
import { t } from "../i18n/index.ts";
|
|
16
16
|
import { DEFAULT_CLAUDE_CODE_CLI_VERSION } from "./coding-cli-versions.ts";
|
|
17
|
-
import
|
|
17
|
+
import { assertMcpServers, isHttpMcp, mcpManifestEntries } from "./mcp.ts";
|
|
18
|
+
import { runPostSetupHooks } from "./post-setup.ts";
|
|
19
|
+
import type { Agent, AgentSetupManifest, McpServer, Sandbox, SandboxHook, SkillSpec } from "../types.ts";
|
|
18
20
|
|
|
19
21
|
// ───────────────────────────────────────────────────────────────────────────
|
|
20
22
|
// Claude Code 的 agent adapter(沙箱型)。
|
|
@@ -69,7 +71,8 @@ export interface ClaudeCodeConfig {
|
|
|
69
71
|
maxTurns?: number;
|
|
70
72
|
/**
|
|
71
73
|
* 额外 MCP server(每个沙箱 setup 时写进用户级 ~/.claude.json)。
|
|
72
|
-
*
|
|
74
|
+
* stdio 形态写 command(可带 args / env);Streamable HTTP 形态写 url(可带 headers,
|
|
75
|
+
* 逐字进请求头),落成 { "type": "http", "url": …, "headers": … } 条目。
|
|
73
76
|
*/
|
|
74
77
|
mcpServers?: McpServer[];
|
|
75
78
|
/**
|
|
@@ -88,6 +91,14 @@ export interface ClaudeCodeConfig {
|
|
|
88
91
|
* setup 报错。manifest 只记项目相对路径与字节 SHA-256,不落正文。
|
|
89
92
|
*/
|
|
90
93
|
settingsFile?: string;
|
|
94
|
+
/**
|
|
95
|
+
* 安装后按数组顺序运行的用户钩子(复用 SandboxHook 的窄上下文):在写 settings、挂 MCP、
|
|
96
|
+
* 装 Skills / Plugin、写 manifest 全部完成后执行,适合跑插件自带的 setup 脚本这类
|
|
97
|
+
* 「安装产物就位后才能跑」的过程动作。钩子返回的 cleanup 按 LIFO 与 teardown 一起收尾;
|
|
98
|
+
* 抛错按基础设施错误计(attempt errored)。
|
|
99
|
+
* 见 docs/feature/adapters/library/coding-agent-extensions.md「安装后运行脚本」。
|
|
100
|
+
*/
|
|
101
|
+
postSetup?: SandboxHook[];
|
|
91
102
|
}
|
|
92
103
|
|
|
93
104
|
export function claudeCodeAgent(config?: ClaudeCodeConfig): Agent {
|
|
@@ -115,7 +126,7 @@ export function claudeCodeAgent(config?: ClaudeCodeConfig): Agent {
|
|
|
115
126
|
}),
|
|
116
127
|
},
|
|
117
128
|
|
|
118
|
-
async setup(sb) {
|
|
129
|
+
async setup(sb, ctx) {
|
|
119
130
|
// 预制模板已把 claude 烘焙进镜像(PATH 上)就跳过安装;否则 npm 全局装。
|
|
120
131
|
await sb.runShell(
|
|
121
132
|
`command -v claude >/dev/null 2>&1 || npm install -g @anthropic-ai/claude-code@${DEFAULT_CLAUDE_CODE_CLI_VERSION}`,
|
|
@@ -140,13 +151,20 @@ export function claudeCodeAgent(config?: ClaudeCodeConfig): Agent {
|
|
|
140
151
|
}
|
|
141
152
|
|
|
142
153
|
if (config?.mcpServers?.length) {
|
|
154
|
+
assertMcpServers(config.mcpServers);
|
|
143
155
|
const servers: Record<string, object> = {};
|
|
144
156
|
for (const s of config.mcpServers) {
|
|
145
|
-
servers[s.name] =
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
157
|
+
servers[s.name] = isHttpMcp(s)
|
|
158
|
+
? {
|
|
159
|
+
type: "http",
|
|
160
|
+
url: s.url,
|
|
161
|
+
...(s.headers && Object.keys(s.headers).length && { headers: s.headers }),
|
|
162
|
+
}
|
|
163
|
+
: {
|
|
164
|
+
command: s.command,
|
|
165
|
+
...(s.args?.length && { args: s.args }),
|
|
166
|
+
...(s.env && { env: s.env }),
|
|
167
|
+
};
|
|
150
168
|
}
|
|
151
169
|
// 用户级 MCP 配置在 ~/.claude.json(顶层 mcpServers 字段),不是 ~/.claude/claude.json
|
|
152
170
|
// ——后者 claude CLI 根本不读,MCP 静默挂不上(本机 `claude mcp list` 可核对)。
|
|
@@ -161,12 +179,8 @@ export function claudeCodeAgent(config?: ClaudeCodeConfig): Agent {
|
|
|
161
179
|
manifest.nativePlugins = await installPlugins(sb, config.plugins);
|
|
162
180
|
}
|
|
163
181
|
if (config?.mcpServers?.length) {
|
|
164
|
-
// manifest 里只记「挂了哪个 server
|
|
165
|
-
manifest.mcpServers = config.mcpServers
|
|
166
|
-
name: s.name,
|
|
167
|
-
command: s.command,
|
|
168
|
-
...(s.args?.length ? { args: [...s.args] } : {}),
|
|
169
|
-
}));
|
|
182
|
+
// manifest 里只记「挂了哪个 server、怎么连」;env / headers 里可能有 token,不落盘。
|
|
183
|
+
manifest.mcpServers = mcpManifestEntries(config.mcpServers);
|
|
170
184
|
}
|
|
171
185
|
if (settings) {
|
|
172
186
|
// 只记来源路径与字节哈希,不落正文(任意官方配置都可能带敏感字符串)。
|
|
@@ -181,6 +195,10 @@ export function claudeCodeAgent(config?: ClaudeCodeConfig): Agent {
|
|
|
181
195
|
) {
|
|
182
196
|
await writeAgentSetupManifest(sb, manifest);
|
|
183
197
|
}
|
|
198
|
+
|
|
199
|
+
// 安装后钩子(postSetup):排在 manifest 之后——manifest 审计 Adapter 自身的安装事实,
|
|
200
|
+
// 钩子失败不该丢掉这份证据。返回的合成 cleanup 交给 runner,与 teardown 一起 LIFO 收尾。
|
|
201
|
+
return await runPostSetupHooks(sb, ctx, config?.postSetup);
|
|
184
202
|
},
|
|
185
203
|
|
|
186
204
|
async send(input, ctx) {
|
package/src/agents/codex.test.ts
CHANGED
|
@@ -17,7 +17,7 @@ import { tmpdir } from "node:os";
|
|
|
17
17
|
import { join } from "node:path";
|
|
18
18
|
import { codexAgent, installPlugins, type CodexPluginSpec } from "./codex.ts";
|
|
19
19
|
import { createAgentSession } from "../context/session.ts";
|
|
20
|
-
import type { AgentContext, AgentSetupManifest, CommandOptions, CommandResult, Sandbox, SandboxFile } from "../types.ts";
|
|
20
|
+
import type { AgentContext, AgentSetupManifest, CommandOptions, CommandResult, McpServer, Sandbox, SandboxFile } from "../types.ts";
|
|
21
21
|
|
|
22
22
|
/** 内存沙箱:runShell 记命令(可按命令包含的子串打脚本化输出),uploadFile / writeFiles 记内容。 */
|
|
23
23
|
class FakeSandbox implements Partial<Sandbox> {
|
|
@@ -91,6 +91,32 @@ describe("codex installPlugins · 命令构造", () => {
|
|
|
91
91
|
]);
|
|
92
92
|
});
|
|
93
93
|
|
|
94
|
+
it("marketplace.sparse 逐项生成 --sparse <path>(缺省不含由上面的精确命令断言覆盖);manifest 不记录 sparse", async () => {
|
|
95
|
+
const box = sb();
|
|
96
|
+
const out = await installPlugins(asSandbox(box), [
|
|
97
|
+
{
|
|
98
|
+
marketplace: {
|
|
99
|
+
name: "acme",
|
|
100
|
+
source: "acme/codex-plugins",
|
|
101
|
+
sparse: [".agents", "plugins/repo-map"],
|
|
102
|
+
},
|
|
103
|
+
name: "repo-map",
|
|
104
|
+
},
|
|
105
|
+
]);
|
|
106
|
+
expect(box.commands[0]).toBe(
|
|
107
|
+
"codex plugin marketplace add 'acme/codex-plugins' --sparse '.agents' --sparse 'plugins/repo-map'",
|
|
108
|
+
);
|
|
109
|
+
expect(out[0]!.marketplace).toEqual({ name: "acme", source: "acme/codex-plugins" });
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
it("marketplace.sparse 空数组与缺省等价:add 命令不含 --sparse", async () => {
|
|
113
|
+
const box = sb();
|
|
114
|
+
await installPlugins(asSandbox(box), [
|
|
115
|
+
{ marketplace: { name: "acme", source: "acme/codex-plugins", sparse: [] }, name: "repo-map" },
|
|
116
|
+
]);
|
|
117
|
+
expect(box.commands[0]).toBe("codex plugin marketplace add 'acme/codex-plugins'");
|
|
118
|
+
});
|
|
119
|
+
|
|
94
120
|
it("同名 marketplace 只连一次:两个 plugin 共用一个 marketplace.name → 只有一条 marketplace add,两条 plugin add", async () => {
|
|
95
121
|
const box = sb();
|
|
96
122
|
const plugins: CodexPluginSpec[] = [
|
|
@@ -339,3 +365,144 @@ describe("codexAgent · live step feedback", () => {
|
|
|
339
365
|
expect(ctx.session.id).toBe("thread-1");
|
|
340
366
|
});
|
|
341
367
|
});
|
|
368
|
+
|
|
369
|
+
describe("codexAgent · exec 信任姿态", () => {
|
|
370
|
+
// bug: memory/codex-hook-trust-headless-silent-skip.md
|
|
371
|
+
it("首轮与 resume 的 exec 命令都含 --dangerously-bypass-hook-trust(headless 下未授信 hook 被 codex 静默跳过,零报错)", async () => {
|
|
372
|
+
const stdout = JSON.stringify({ type: "thread.started", thread_id: "thread-1" }) + "\n";
|
|
373
|
+
const box = sb([{ match: "codex exec", result: () => ({ stdout }) }]);
|
|
374
|
+
const ctx: AgentContext = {
|
|
375
|
+
signal: new AbortController().signal,
|
|
376
|
+
flags: {},
|
|
377
|
+
sandbox: asSandbox(box),
|
|
378
|
+
session: createAgentSession(),
|
|
379
|
+
progress() {},
|
|
380
|
+
diagnostic() {},
|
|
381
|
+
log() {},
|
|
382
|
+
};
|
|
383
|
+
const agent = codexAgent({ apiKey: "test-key" });
|
|
384
|
+
|
|
385
|
+
await agent.send!({ text: "first" }, ctx);
|
|
386
|
+
await agent.send!({ text: "second" }, ctx);
|
|
387
|
+
|
|
388
|
+
const execs = box.commands.filter((c) => c.startsWith("codex exec"));
|
|
389
|
+
expect(execs).toHaveLength(2);
|
|
390
|
+
expect(execs[0]).not.toContain("codex exec resume");
|
|
391
|
+
expect(execs[1]).toContain("codex exec resume thread-1");
|
|
392
|
+
for (const cmd of execs) expect(cmd).toContain("--dangerously-bypass-hook-trust");
|
|
393
|
+
});
|
|
394
|
+
});
|
|
395
|
+
|
|
396
|
+
describe("codexAgent mcpServers · 形态落位", () => {
|
|
397
|
+
const ctx = { flags: {} } as AgentContext;
|
|
398
|
+
|
|
399
|
+
it("HTTP 形态:url 行 + [mcp_servers.<name>.http_headers] 子表;manifest 只记 name/url,headers 值不落盘", async () => {
|
|
400
|
+
const box = sb();
|
|
401
|
+
await codexAgent({
|
|
402
|
+
apiKey: "k",
|
|
403
|
+
mcpServers: [
|
|
404
|
+
{ name: "team-memory", url: "https://mem.example.com/mcp/", headers: { Authorization: "Bearer sekret" } },
|
|
405
|
+
],
|
|
406
|
+
}).setup!(asSandbox(box), ctx);
|
|
407
|
+
|
|
408
|
+
const mcp = box.commands.find((c) => c.includes("[mcp_servers.team-memory]"))!;
|
|
409
|
+
expect(mcp).toContain('url = "https://mem.example.com/mcp/"');
|
|
410
|
+
expect(mcp).toContain("[mcp_servers.team-memory.http_headers]");
|
|
411
|
+
expect(mcp).toContain('"Authorization" = "Bearer sekret"');
|
|
412
|
+
expect(mcp).not.toContain("command =");
|
|
413
|
+
|
|
414
|
+
const manifestRaw = box.written["__niceeval__/agent-setup.json"]!;
|
|
415
|
+
const manifest = JSON.parse(manifestRaw) as AgentSetupManifest;
|
|
416
|
+
expect(manifest.mcpServers).toEqual([{ name: "team-memory", url: "https://mem.example.com/mcp/" }]);
|
|
417
|
+
expect(manifestRaw).not.toContain("sekret");
|
|
418
|
+
});
|
|
419
|
+
|
|
420
|
+
it("边界:HTTP 形态无 headers → 不写空 http_headers 子表", async () => {
|
|
421
|
+
const box = sb();
|
|
422
|
+
await codexAgent({
|
|
423
|
+
apiKey: "k",
|
|
424
|
+
mcpServers: [{ name: "team-memory", url: "https://mem.example.com/mcp/" }],
|
|
425
|
+
}).setup!(asSandbox(box), ctx);
|
|
426
|
+
|
|
427
|
+
const mcp = box.commands.find((c) => c.includes("[mcp_servers.team-memory]"))!;
|
|
428
|
+
expect(mcp).toContain('url = "https://mem.example.com/mcp/"');
|
|
429
|
+
expect(mcp).not.toContain("http_headers");
|
|
430
|
+
});
|
|
431
|
+
|
|
432
|
+
it("反例:同一 server 同时给出 command 与 url → setup 报错点名该 server,不写 MCP 块", async () => {
|
|
433
|
+
const box = sb();
|
|
434
|
+
// 形状判别不设 kind 标签,双字段的错误配置在类型上可能混得进来 —— 运行期兜底点名报错。
|
|
435
|
+
const dup = { name: "dup-server", command: "npx", url: "https://x.example.com/mcp" } as McpServer;
|
|
436
|
+
await expect(codexAgent({ apiKey: "k", mcpServers: [dup] }).setup!(asSandbox(box), ctx)).rejects.toThrow(
|
|
437
|
+
/dup-server/,
|
|
438
|
+
);
|
|
439
|
+
expect(box.commands.some((c) => c.includes("[mcp_servers."))).toBe(false);
|
|
440
|
+
});
|
|
441
|
+
});
|
|
442
|
+
|
|
443
|
+
describe("codexAgent postSetup · 安装后钩子", () => {
|
|
444
|
+
const mkCtx = (): AgentContext =>
|
|
445
|
+
({
|
|
446
|
+
flags: {},
|
|
447
|
+
experimentId: "exp-1",
|
|
448
|
+
signal: new AbortController().signal,
|
|
449
|
+
progress: () => {},
|
|
450
|
+
diagnostic: () => {},
|
|
451
|
+
}) as unknown as AgentContext;
|
|
452
|
+
|
|
453
|
+
it("钩子在 Adapter 安装与 manifest 之后按数组顺序执行,拿到 SandboxHook 窄上下文", async () => {
|
|
454
|
+
const box = sb();
|
|
455
|
+
const seen: string[] = [];
|
|
456
|
+
let manifestPresentAtHook = false;
|
|
457
|
+
await codexAgent({
|
|
458
|
+
apiKey: "k",
|
|
459
|
+
mcpServers: [{ name: "browser", command: "npx" }],
|
|
460
|
+
postSetup: [
|
|
461
|
+
async (sandbox, hookCtx) => {
|
|
462
|
+
manifestPresentAtHook = box.written["__niceeval__/agent-setup.json"] !== undefined;
|
|
463
|
+
seen.push(`a:${hookCtx.experimentId}`);
|
|
464
|
+
await sandbox.runShell("post-hook-a");
|
|
465
|
+
},
|
|
466
|
+
async (sandbox) => {
|
|
467
|
+
seen.push("b");
|
|
468
|
+
await sandbox.runShell("post-hook-b");
|
|
469
|
+
},
|
|
470
|
+
],
|
|
471
|
+
}).setup!(asSandbox(box), mkCtx());
|
|
472
|
+
|
|
473
|
+
expect(seen).toEqual(["a:exp-1", "b"]);
|
|
474
|
+
expect(manifestPresentAtHook).toBe(true);
|
|
475
|
+
const configIdx = box.commands.findIndex((c) => c.includes("config.toml"));
|
|
476
|
+
const aIdx = box.commands.indexOf("post-hook-a");
|
|
477
|
+
const bIdx = box.commands.indexOf("post-hook-b");
|
|
478
|
+
expect(aIdx).toBeGreaterThan(configIdx);
|
|
479
|
+
expect(bIdx).toBeGreaterThan(aIdx);
|
|
480
|
+
});
|
|
481
|
+
|
|
482
|
+
it("钩子返回的 cleanup 合成一个闭包交还 runner,按 LIFO 执行", async () => {
|
|
483
|
+
const box = sb();
|
|
484
|
+
const order: string[] = [];
|
|
485
|
+
const cleanup = await codexAgent({
|
|
486
|
+
apiKey: "k",
|
|
487
|
+
postSetup: [() => () => void order.push("a"), () => () => void order.push("b")],
|
|
488
|
+
}).setup!(asSandbox(box), mkCtx());
|
|
489
|
+
|
|
490
|
+
expect(typeof cleanup).toBe("function");
|
|
491
|
+
await (cleanup as () => Promise<void> | void)();
|
|
492
|
+
expect(order).toEqual(["b", "a"]);
|
|
493
|
+
});
|
|
494
|
+
|
|
495
|
+
it("反例:钩子抛错从 setup 传播(attempt errored 通道)", async () => {
|
|
496
|
+
const box = sb();
|
|
497
|
+
await expect(
|
|
498
|
+
codexAgent({
|
|
499
|
+
apiKey: "k",
|
|
500
|
+
postSetup: [
|
|
501
|
+
() => {
|
|
502
|
+
throw new Error("hook boom");
|
|
503
|
+
},
|
|
504
|
+
],
|
|
505
|
+
}).setup!(asSandbox(box), mkCtx()),
|
|
506
|
+
).rejects.toThrow("hook boom");
|
|
507
|
+
});
|
|
508
|
+
});
|
package/src/agents/codex.ts
CHANGED
|
@@ -19,7 +19,9 @@ import {
|
|
|
19
19
|
import { mapCodexSpans } from "../o11y/otlp/mappers/codex.ts";
|
|
20
20
|
import { t } from "../i18n/index.ts";
|
|
21
21
|
import { DEFAULT_CODEX_CLI_VERSION } from "./coding-cli-versions.ts";
|
|
22
|
-
import
|
|
22
|
+
import { assertMcpServers, isHttpMcp, mcpManifestEntries } from "./mcp.ts";
|
|
23
|
+
import { runPostSetupHooks } from "./post-setup.ts";
|
|
24
|
+
import type { Agent, AgentSetupManifest, McpServer, Sandbox, SandboxHook, SkillSpec } from "../types.ts";
|
|
23
25
|
|
|
24
26
|
// ───────────────────────────────────────────────────────────────────────────
|
|
25
27
|
// OpenAI Codex CLI 的 agent adapter(沙箱型)。
|
|
@@ -59,6 +61,12 @@ export interface CodexPluginSpec {
|
|
|
59
61
|
source: string;
|
|
60
62
|
/** 固定 Marketplace 的 Tag、Commit 或 Branch(→ `codex plugin marketplace add --ref`)。 */
|
|
61
63
|
ref?: string;
|
|
64
|
+
/**
|
|
65
|
+
* sparse 拉取的路径列表,每项生成一个 `codex plugin marketplace add --sparse <path>`
|
|
66
|
+
* (codex 的 `--sparse` 必须带路径参数、可重复):大仓库只取插件所需路径,省略或空数组即全量 clone。
|
|
67
|
+
* 只影响拉取速度,不影响装出来的内容;manifest 不记录它。
|
|
68
|
+
*/
|
|
69
|
+
sparse?: string[];
|
|
62
70
|
};
|
|
63
71
|
/** Marketplace 中的 Plugin 名。 */
|
|
64
72
|
name: string;
|
|
@@ -71,7 +79,8 @@ export interface CodexConfig {
|
|
|
71
79
|
baseUrl?: string;
|
|
72
80
|
/**
|
|
73
81
|
* 额外 MCP server(每个沙箱 setup 时追加进 ~/.codex/config.toml)。
|
|
74
|
-
*
|
|
82
|
+
* stdio 形态(command/args/env)写 [mcp_servers.<name>] 的 command 行;
|
|
83
|
+
* Streamable HTTP 形态(url/headers)写 url 行,headers 进 [mcp_servers.<name>.http_headers] 子表。
|
|
75
84
|
*/
|
|
76
85
|
mcpServers?: McpServer[];
|
|
77
86
|
/**
|
|
@@ -92,6 +101,14 @@ export interface CodexConfig {
|
|
|
92
101
|
* 项目相对路径与字节 SHA-256,不落正文。
|
|
93
102
|
*/
|
|
94
103
|
configFile?: string;
|
|
104
|
+
/**
|
|
105
|
+
* 安装后按数组顺序运行的用户钩子(复用 SandboxHook 的窄上下文):在写主配置、挂 MCP、
|
|
106
|
+
* 装 Skills / Plugin、写 manifest 全部完成后执行,适合跑插件自带的 setup 脚本这类
|
|
107
|
+
* 「安装产物就位后才能跑」的过程动作。钩子返回的 cleanup 按 LIFO 与 teardown 一起收尾;
|
|
108
|
+
* 抛错按基础设施错误计(attempt errored)。
|
|
109
|
+
* 见 docs/feature/adapters/library/coding-agent-extensions.md「安装后运行脚本」。
|
|
110
|
+
*/
|
|
111
|
+
postSetup?: SandboxHook[];
|
|
95
112
|
}
|
|
96
113
|
|
|
97
114
|
export function codexAgent(config?: CodexConfig): Agent {
|
|
@@ -155,15 +172,25 @@ export function codexAgent(config?: CodexConfig): Agent {
|
|
|
155
172
|
}
|
|
156
173
|
|
|
157
174
|
if (config?.mcpServers?.length) {
|
|
175
|
+
assertMcpServers(config.mcpServers);
|
|
158
176
|
const mcpToml = config.mcpServers
|
|
159
177
|
.map((s) => {
|
|
160
178
|
// 注意是复数 mcp_servers:单数 [mcp_server.x] 会被 codex 静默忽略,
|
|
161
179
|
// MCP 压根挂不上(实测 codex-cli 0.142.x,`codex mcp list` 可核对)。
|
|
162
|
-
const lines: string[] = [`[mcp_servers.${s.name}]
|
|
163
|
-
if (
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
180
|
+
const lines: string[] = [`[mcp_servers.${s.name}]`];
|
|
181
|
+
if (isHttpMcp(s)) {
|
|
182
|
+
lines.push(`url = "${s.url}"`);
|
|
183
|
+
if (s.headers && Object.keys(s.headers).length) {
|
|
184
|
+
lines.push(`[mcp_servers.${s.name}.http_headers]`);
|
|
185
|
+
for (const [k, v] of Object.entries(s.headers)) lines.push(`"${k}" = "${v}"`);
|
|
186
|
+
}
|
|
187
|
+
} else {
|
|
188
|
+
lines.push(`command = "${s.command}"`);
|
|
189
|
+
if (s.args?.length) lines.push(`args = [${s.args.map((a) => `"${a}"`).join(", ")}]`);
|
|
190
|
+
if (s.env && Object.keys(s.env).length) {
|
|
191
|
+
lines.push(`[mcp_servers.${s.name}.env]`);
|
|
192
|
+
for (const [k, v] of Object.entries(s.env)) lines.push(`${k} = "${v}"`);
|
|
193
|
+
}
|
|
167
194
|
}
|
|
168
195
|
return lines.join("\n");
|
|
169
196
|
})
|
|
@@ -184,12 +211,8 @@ export function codexAgent(config?: CodexConfig): Agent {
|
|
|
184
211
|
manifest.nativePlugins = await installPlugins(sb, config.plugins);
|
|
185
212
|
}
|
|
186
213
|
if (config?.mcpServers?.length) {
|
|
187
|
-
// manifest 只记「挂了哪个 server
|
|
188
|
-
manifest.mcpServers = config.mcpServers
|
|
189
|
-
name: s.name,
|
|
190
|
-
command: s.command,
|
|
191
|
-
...(s.args?.length ? { args: [...s.args] } : {}),
|
|
192
|
-
}));
|
|
214
|
+
// manifest 只记「挂了哪个 server、怎么连」;env / headers 里可能有 token,不落盘。
|
|
215
|
+
manifest.mcpServers = mcpManifestEntries(config.mcpServers);
|
|
193
216
|
}
|
|
194
217
|
if (nativeConfig) {
|
|
195
218
|
// 只记来源路径与字节哈希,不落正文(任意官方配置都可能带敏感字符串)。
|
|
@@ -203,6 +226,10 @@ export function codexAgent(config?: CodexConfig): Agent {
|
|
|
203
226
|
) {
|
|
204
227
|
await writeAgentSetupManifest(sb, manifest);
|
|
205
228
|
}
|
|
229
|
+
|
|
230
|
+
// 安装后钩子(postSetup):排在 manifest 之后——manifest 审计 Adapter 自身的安装事实,
|
|
231
|
+
// 钩子失败不该丢掉这份证据。返回的合成 cleanup 交给 runner,与 teardown 一起 LIFO 收尾。
|
|
232
|
+
return await runPostSetupHooks(sb, ctx, config?.postSetup);
|
|
206
233
|
},
|
|
207
234
|
|
|
208
235
|
tracing: {
|
|
@@ -223,7 +250,12 @@ export function codexAgent(config?: CodexConfig): Agent {
|
|
|
223
250
|
|
|
224
251
|
async send(input, ctx) {
|
|
225
252
|
const sb = ctx.sandbox;
|
|
226
|
-
|
|
253
|
+
// hook trust bypass 是 runtime-only(config.toml 设不了):headless 下 codex 对非 managed
|
|
254
|
+
// 来源 hook 的交互式授信永远无人应答,不带它插件装的 hook 会被静默跳过、零报错
|
|
255
|
+
// (见 memory/codex-hook-trust-headless-silent-skip.md)。沙箱内 hook 来源全部由实验配置
|
|
256
|
+
// 声明,与 approvals/sandbox bypass 同一信任层级。
|
|
257
|
+
const flags =
|
|
258
|
+
"--json --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check --dangerously-bypass-hook-trust";
|
|
227
259
|
const prompt = shared.shellQuote(input.text);
|
|
228
260
|
const resuming = ctx.session.id;
|
|
229
261
|
const cmd = resuming
|
|
@@ -324,8 +356,12 @@ export async function installPlugins(
|
|
|
324
356
|
const { marketplace } = plugin;
|
|
325
357
|
if (!connected.has(marketplace.name)) {
|
|
326
358
|
const refFlag = marketplace.ref ? ` --ref ${shared.shellQuote(marketplace.ref)}` : "";
|
|
359
|
+
// --sparse 只影响拉取速度,不影响装出来的内容;manifest 不记录它。
|
|
360
|
+
const sparseFlags = (marketplace.sparse ?? [])
|
|
361
|
+
.map((path) => ` --sparse ${shared.shellQuote(path)}`)
|
|
362
|
+
.join("");
|
|
327
363
|
const add = await sb.runShell(
|
|
328
|
-
`codex plugin marketplace add ${shared.shellQuote(marketplace.source)}${refFlag}`,
|
|
364
|
+
`codex plugin marketplace add ${shared.shellQuote(marketplace.source)}${refFlag}${sparseFlags}`,
|
|
329
365
|
);
|
|
330
366
|
if (add.exitCode !== 0) {
|
|
331
367
|
throw new Error(
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// MCP server 的形态判别与 manifest 映射 —— Claude Code / Codex 两个 adapter 共用。
|
|
2
|
+
// 契约见 docs/feature/adapters/architecture/coding-agent-extensions.md「类型边界」:
|
|
3
|
+
// stdio(command)与 Streamable HTTP(url)按形状判别;双字段是配置错误,setup 点名报错。
|
|
4
|
+
|
|
5
|
+
import { t } from "../i18n/index.ts";
|
|
6
|
+
import type { AgentSetupManifest, McpHttpServer, McpServer } from "./types.ts";
|
|
7
|
+
|
|
8
|
+
/** 形态判别:有 url 的是 HTTP。调用前先过 {@link assertMcpServers}(双字段在那里报错)。 */
|
|
9
|
+
export function isHttpMcp(server: McpServer): server is McpHttpServer {
|
|
10
|
+
return "url" in server && typeof server.url === "string";
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** 双字段(command + url)的配置错误在写任何沙箱配置前抛出,点名 server。 */
|
|
14
|
+
export function assertMcpServers(servers: readonly McpServer[]): void {
|
|
15
|
+
for (const server of servers) {
|
|
16
|
+
if ("command" in server && "url" in server) {
|
|
17
|
+
throw new Error(t("mcp.ambiguousTransport", { name: server.name }));
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** manifest 条目:只记非 secret 字段(stdio 不含 env,HTTP 不含 headers)。 */
|
|
23
|
+
export function mcpManifestEntries(
|
|
24
|
+
servers: readonly McpServer[],
|
|
25
|
+
): NonNullable<AgentSetupManifest["mcpServers"]> {
|
|
26
|
+
return servers.map((s) =>
|
|
27
|
+
isHttpMcp(s)
|
|
28
|
+
? { name: s.name, url: s.url }
|
|
29
|
+
: { name: s.name, command: s.command, ...(s.args?.length ? { args: [...s.args] } : {}) },
|
|
30
|
+
);
|
|
31
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// factory 的 postSetup 钩子执行器 —— Claude Code / Codex / Bub 共用。
|
|
2
|
+
// 契约见 docs/feature/adapters/library/coding-agent-extensions.md「安装后运行脚本」:
|
|
3
|
+
// 在 adapter 全部安装步骤(含 manifest)之后按数组顺序执行;复用 SandboxHook 的窄上下文;
|
|
4
|
+
// 返回的 cleanup 合成一个 LIFO 闭包,由 runner 与 agent teardown 一起收尾;钩子抛错
|
|
5
|
+
// 直接传播(attempt errored)。
|
|
6
|
+
|
|
7
|
+
import type { Cleanup } from "../shared/types.ts";
|
|
8
|
+
import type { Sandbox, SandboxHook, SandboxHookContext } from "../sandbox/types.ts";
|
|
9
|
+
import type { AgentContext } from "./types.ts";
|
|
10
|
+
|
|
11
|
+
export async function runPostSetupHooks(
|
|
12
|
+
sb: Sandbox,
|
|
13
|
+
ctx: AgentContext,
|
|
14
|
+
hooks: readonly SandboxHook[] | undefined,
|
|
15
|
+
): Promise<Cleanup | void> {
|
|
16
|
+
if (!hooks?.length) return;
|
|
17
|
+
// 窄上下文与沙箱钩子同款:不把 session / model / telemetry 借给过程钩子。
|
|
18
|
+
const hookCtx: SandboxHookContext = {
|
|
19
|
+
experimentId: ctx.experimentId,
|
|
20
|
+
signal: ctx.signal,
|
|
21
|
+
progress: (update) => ctx.progress(update),
|
|
22
|
+
diagnostic: (input) => ctx.diagnostic(input),
|
|
23
|
+
};
|
|
24
|
+
const cleanups: Cleanup[] = [];
|
|
25
|
+
for (const hook of hooks) {
|
|
26
|
+
const cleanup = await hook(sb, hookCtx);
|
|
27
|
+
if (typeof cleanup === "function") cleanups.push(cleanup);
|
|
28
|
+
}
|
|
29
|
+
if (!cleanups.length) return;
|
|
30
|
+
return async () => {
|
|
31
|
+
for (const cleanup of [...cleanups].reverse()) await cleanup();
|
|
32
|
+
};
|
|
33
|
+
}
|
package/src/agents/types.ts
CHANGED
|
@@ -7,21 +7,42 @@ import type { StreamEvent, TraceSpan, Usage } from "../o11y/types.ts";
|
|
|
7
7
|
import type { Sandbox } from "../sandbox/types.ts";
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* 见 docs/feature/adapters/architecture/coding-agent-extensions.md「类型边界」。
|
|
10
|
+
* 本地 stdio 形态的 MCP server:沙箱内起子进程,按 stdio 说 MCP 协议。
|
|
11
|
+
* 与 {@link McpHttpServer} 按形状判别(有 `command` 的是 stdio,有 `url` 的是 HTTP)。
|
|
13
12
|
*/
|
|
14
|
-
export interface
|
|
13
|
+
export interface McpStdioServer {
|
|
15
14
|
/** 服务器唯一名(config key)。 */
|
|
16
15
|
name: string;
|
|
17
16
|
/** 启动命令(如 "npx"、"node"、"uvx")。 */
|
|
18
17
|
command: string;
|
|
19
18
|
/** 传给命令的参数。 */
|
|
20
19
|
args?: string[];
|
|
21
|
-
/**
|
|
20
|
+
/** 注入服务器进程的环境变量(可能含 secret,不进 manifest)。 */
|
|
22
21
|
env?: Record<string, string>;
|
|
23
22
|
}
|
|
24
23
|
|
|
24
|
+
/**
|
|
25
|
+
* 远程 Streamable HTTP 形态的 MCP server:沙箱直接连一个 HTTP 端点。
|
|
26
|
+
* `url` 必须沙箱内可达——宿主机上的服务先经隧道(cloudflared / tailscale 等)暴露。
|
|
27
|
+
*/
|
|
28
|
+
export interface McpHttpServer {
|
|
29
|
+
/** 服务器唯一名(config key)。 */
|
|
30
|
+
name: string;
|
|
31
|
+
/** Streamable HTTP 端点(如 https://mem.example.com/mcp/)。 */
|
|
32
|
+
url: string;
|
|
33
|
+
/** 逐字写进每个请求的 HTTP 头(常用于 Authorization;可能含 secret,不进 manifest)。 */
|
|
34
|
+
headers?: Record<string, string>;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* MCP server 描述符 —— 支持 MCP 的 adapter(Claude Code / Codex)共用的工具服务单元,
|
|
39
|
+
* 不是 native plugin 的一种。stdio 与 Streamable HTTP 两种形态按形状判别,不设 kind 标签
|
|
40
|
+
* (两种形态各有唯一必填判别字段);同时给出 `command` 与 `url` 属配置错误,setup 报错点名。
|
|
41
|
+
* 在 agent factory config 里声明,setup 阶段写进各自的配置文件。
|
|
42
|
+
* 见 docs/feature/adapters/architecture/coding-agent-extensions.md「类型边界」。
|
|
43
|
+
*/
|
|
44
|
+
export type McpServer = McpStdioServer | McpHttpServer;
|
|
45
|
+
|
|
25
46
|
/**
|
|
26
47
|
* Skill 的来源描述 —— Claude Code / Codex / Bub 共用的**数据类型**:只统一「从哪里取得
|
|
27
48
|
* 哪份 Skill」,安装位置、发现机制、要不要额外写 project instruction 由各 Adapter 决定。
|
|
@@ -67,8 +88,8 @@ export interface AgentSetupManifest {
|
|
|
67
88
|
/** 安装后 CLI 报告的版本;取不到时省略。 */
|
|
68
89
|
resolvedVersion?: string;
|
|
69
90
|
}>;
|
|
70
|
-
/** 挂上的 MCP server(不含 env
|
|
71
|
-
mcpServers?: Array<{ name: string; command: string; args?: string[] }>;
|
|
91
|
+
/** 挂上的 MCP server(只记非 secret 字段:stdio 不含 env,HTTP 不含 headers)。 */
|
|
92
|
+
mcpServers?: Array<{ name: string; command: string; args?: string[] } | { name: string; url: string }>;
|
|
72
93
|
/**
|
|
73
94
|
* 官方原生配置文件(Claude Code `settings.json` / Codex `config.toml`):只记 Agent 名、
|
|
74
95
|
* 项目相对来源路径与原始字节的 SHA-256,不落正文 —— 任意官方配置都可能携带敏感字符串,
|