niceeval 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/README.zh.md +13 -13
- package/docs/README.md +119 -0
- package/docs/adapters/README.md +60 -0
- package/docs/adapters/authoring.md +179 -0
- package/docs/adapters/coding-agent-skills-plugins.md +324 -0
- package/docs/adapters/collection.md +128 -0
- package/docs/adapters/contract.md +263 -0
- package/docs/adapters/reference/agent-eval.md +215 -0
- package/docs/adapters/reference/agent-loop-apis.md +69 -0
- package/docs/adapters/reference/claude-code-otel-telemetry.md +100 -0
- package/docs/adapters/reference/eve-protocol.md +127 -0
- package/docs/adapters/reference/otel-genai.md +107 -0
- package/docs/adapters/reference/otel-instrumentation.md +65 -0
- package/docs/adapters/targets.md +99 -0
- package/docs/architecture.md +140 -0
- package/docs/assertions.md +388 -0
- package/docs/capabilities-by-construction.md +48 -0
- package/docs/cli.md +171 -0
- package/docs/concepts.md +106 -0
- package/docs/e2e-ci.md +295 -0
- package/docs/eval-authoring.md +319 -0
- package/docs/experiments.md +154 -0
- package/docs/getting-started.md +224 -0
- package/docs/multi-agent.md +145 -0
- package/docs/observability.md +333 -0
- package/docs/origin-integration.md +195 -0
- package/docs/references.md +35 -0
- package/docs/results-format.md +228 -0
- package/docs/runner.md +104 -0
- package/docs/sandbox.md +276 -0
- package/docs/scoring.md +119 -0
- package/docs/source-map.md +106 -0
- package/docs/tier-sync.md +177 -0
- package/docs/view.md +157 -0
- package/docs/vision.md +88 -0
- package/package.json +35 -5
- package/src/agents/ai-sdk-otel.ts +0 -0
- package/src/agents/ai-sdk.test.ts +377 -0
- package/src/agents/ai-sdk.ts +577 -0
- package/src/agents/bub.ts +30 -37
- package/src/agents/claude-code.ts +7 -4
- package/src/agents/codex.ts +15 -10
- package/src/agents/index.ts +43 -1
- package/src/agents/sdk-streams.test.ts +145 -0
- package/src/agents/sdk-streams.ts +457 -0
- package/src/agents/shared.ts +77 -39
- package/src/agents/streaming.test.ts +152 -0
- package/src/agents/streaming.ts +195 -0
- package/src/agents/types.ts +218 -0
- package/src/agents/ui-message-stream.test.ts +249 -0
- package/src/agents/ui-message-stream.ts +372 -0
- package/src/cli.ts +124 -77
- package/src/context/context.test.ts +203 -7
- package/src/context/context.ts +158 -49
- package/src/context/session.test.ts +96 -0
- package/src/context/session.ts +119 -9
- package/src/context/types.ts +205 -0
- package/src/define.ts +4 -14
- package/src/expect/index.ts +8 -71
- package/src/i18n/core.ts +28 -0
- package/src/i18n/en.ts +47 -5
- package/src/i18n/index.ts +5 -17
- package/src/i18n/zh-CN.ts +47 -5
- package/src/index.ts +12 -0
- package/src/loaders/index.ts +8 -39
- package/src/o11y/otlp/canonical.ts +7 -6
- package/src/o11y/otlp/mappers/codex.ts +10 -8
- package/src/o11y/otlp/mappers/index.ts +9 -21
- package/src/o11y/otlp/receiver.ts +25 -7
- package/src/o11y/otlp/sandbox-receiver.ts +57 -21
- package/src/o11y/otlp/turn-otel.test.ts +110 -0
- package/src/o11y/otlp/turn-otel.ts +125 -0
- package/src/o11y/parsers/bub.ts +13 -11
- package/src/o11y/parsers/claude-code.ts +27 -51
- package/src/o11y/parsers/codex.ts +29 -46
- package/src/o11y/parsers/index.ts +5 -14
- package/src/o11y/tool-names.test.ts +61 -0
- package/src/o11y/tool-names.ts +74 -0
- package/src/o11y/types.ts +135 -0
- package/src/runner/attempt.ts +456 -0
- package/src/runner/fingerprint.ts +59 -0
- package/src/runner/remote-sandbox.ts +53 -0
- package/src/runner/report.ts +53 -0
- package/src/runner/reporters/artifacts.ts +25 -4
- package/src/runner/reporters/console.ts +2 -8
- package/src/runner/reporters/live.ts +2 -8
- package/src/runner/reporters/shared.ts +15 -0
- package/src/runner/reporters/table.ts +10 -62
- package/src/runner/run.ts +114 -619
- package/src/runner/types.ts +237 -0
- package/src/sandbox/checkpoint.ts +15 -13
- package/src/sandbox/docker-stream.ts +87 -0
- package/src/sandbox/docker.ts +42 -120
- package/src/sandbox/e2b.ts +24 -75
- package/src/sandbox/local-files.ts +33 -0
- package/src/sandbox/paths.test.ts +85 -0
- package/src/sandbox/paths.ts +45 -0
- package/src/sandbox/resolve.ts +10 -34
- package/src/sandbox/shell.ts +17 -0
- package/src/sandbox/source-files.ts +42 -2
- package/src/sandbox/types.ts +158 -0
- package/src/sandbox/vercel.ts +55 -71
- package/src/scoring/collector.ts +3 -2
- package/src/scoring/judge.ts +72 -146
- package/src/scoring/match.ts +79 -0
- package/src/scoring/types.ts +76 -0
- package/src/shared/aggregate.ts +48 -0
- package/src/shared/format.ts +20 -0
- package/src/shared/outcome.ts +48 -0
- package/src/shared/types.ts +37 -0
- package/src/types.ts +20 -911
- package/src/view/aggregate.ts +103 -0
- package/src/view/app/App.tsx +26 -5
- package/src/view/app/components/AttemptModal.tsx +12 -3
- package/src/view/app/components/CodeView.tsx +9 -5
- package/src/view/app/components/CopyControls.tsx +4 -4
- package/src/view/app/components/ExperimentTable.tsx +5 -5
- package/src/view/app/i18n.ts +31 -7
- package/src/view/app/lib/format.ts +17 -20
- package/src/view/app/lib/outcome.ts +4 -16
- package/src/view/app/main.tsx +3 -5
- package/src/view/app/pages/RunsPage.tsx +2 -2
- package/src/view/app/pages/TracesPage.tsx +2 -2
- package/src/view/app/shared.ts +2 -1
- package/src/view/app/types.ts +6 -43
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +18 -18
- package/src/view/index.ts +24 -401
- package/src/view/loader.ts +234 -0
- package/src/view/server.ts +136 -0
- package/src/view/shared/types.ts +64 -0
- package/src/view/styles.css +60 -9
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
|
|
3
|
+
import { deltaStream, driveFrameStream } from "./streaming.ts";
|
|
4
|
+
import type { DeltaOp } from "./streaming.ts";
|
|
5
|
+
import type { AgentContext } from "../types.ts";
|
|
6
|
+
import { createAgentSession } from "../context/session.ts";
|
|
7
|
+
|
|
8
|
+
/** 一条会话线 = 一个 ctx.session。同一个 ctx 重复用 = 同一条线;新造一个 = 新线。 */
|
|
9
|
+
function lineCtx(): AgentContext {
|
|
10
|
+
return {
|
|
11
|
+
signal: new AbortController().signal,
|
|
12
|
+
flags: {},
|
|
13
|
+
session: createAgentSession(),
|
|
14
|
+
sandbox: undefined as never,
|
|
15
|
+
log() {},
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
describe("deltaStream", () => {
|
|
20
|
+
// 模拟一个 OpenAI Chat Completions 风格的原始增量流:choices[0].delta.content 逐 token、
|
|
21
|
+
// choices[0].delta.tool_calls[i] 逐参数、finish_reason 收尾、独立的 usage 帧。
|
|
22
|
+
interface RawChunk {
|
|
23
|
+
delta?: { content?: string; tool_calls?: { index: number; id?: string; function?: { name?: string; arguments?: string } }[] };
|
|
24
|
+
finish_reason?: string | null;
|
|
25
|
+
usage?: { prompt_tokens: number; completion_tokens: number };
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const toolIndexToCallId = new Map<number, string>();
|
|
29
|
+
|
|
30
|
+
function toOps(chunk: RawChunk): DeltaOp[] {
|
|
31
|
+
const ops: DeltaOp[] = [];
|
|
32
|
+
if (chunk.delta?.content) ops.push({ kind: "text-delta", text: chunk.delta.content });
|
|
33
|
+
for (const tc of chunk.delta?.tool_calls ?? []) {
|
|
34
|
+
if (tc.id) {
|
|
35
|
+
toolIndexToCallId.set(tc.index, tc.id);
|
|
36
|
+
ops.push({ kind: "tool-call-start", callId: tc.id, name: tc.function?.name ?? "unknown" });
|
|
37
|
+
}
|
|
38
|
+
const callId = toolIndexToCallId.get(tc.index)!;
|
|
39
|
+
if (tc.function?.arguments) ops.push({ kind: "tool-args-delta", callId, delta: tc.function.arguments });
|
|
40
|
+
}
|
|
41
|
+
if (chunk.finish_reason === "tool_calls") {
|
|
42
|
+
for (const callId of toolIndexToCallId.values()) ops.push({ kind: "tool-call-end", callId });
|
|
43
|
+
}
|
|
44
|
+
if (chunk.finish_reason === "stop") ops.push({ kind: "message-end" });
|
|
45
|
+
if (chunk.usage) ops.push({ kind: "usage", usage: { inputTokens: chunk.usage.prompt_tokens, outputTokens: chunk.usage.completion_tokens } });
|
|
46
|
+
return ops;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
it("逐 token 文本拼接,message-end 落地成一条 message", () => {
|
|
50
|
+
const s = deltaStream({ toOps });
|
|
51
|
+
expect(s.add({ delta: { content: "北" } })).toEqual([]);
|
|
52
|
+
expect(s.add({ delta: { content: "京" } })).toEqual([]);
|
|
53
|
+
expect(s.add({ delta: { content: "晴" } })).toEqual([]);
|
|
54
|
+
expect(s.add({ finish_reason: "stop" })).toEqual([{ type: "message", role: "assistant", text: "北京晴" }]);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it("逐参数拼接 + 合法 JSON 落地成 action.called;usage 帧旁路累积", () => {
|
|
58
|
+
toolIndexToCallId.clear();
|
|
59
|
+
const s = deltaStream({ toOps });
|
|
60
|
+
s.add({ delta: { tool_calls: [{ index: 0, id: "call_1", function: { name: "get_weather", arguments: '{"cit' } }] } });
|
|
61
|
+
s.add({ delta: { tool_calls: [{ index: 0, function: { arguments: 'y":"北京"}' } }] } });
|
|
62
|
+
const events = s.add({ finish_reason: "tool_calls" });
|
|
63
|
+
expect(events).toEqual([{ type: "action.called", callId: "call_1", name: "get_weather", input: { city: "北京" } }]);
|
|
64
|
+
|
|
65
|
+
s.add({ usage: { prompt_tokens: 42, completion_tokens: 7 } });
|
|
66
|
+
expect(s.usage).toEqual({ inputTokens: 42, outputTokens: 7 });
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it("拼不出合法 JSON 时,把原始字符串塞进 input,不吞错误", () => {
|
|
70
|
+
toolIndexToCallId.clear();
|
|
71
|
+
const s = deltaStream({ toOps });
|
|
72
|
+
s.add({ delta: { tool_calls: [{ index: 0, id: "call_2", function: { name: "broken", arguments: "not json" } }] } });
|
|
73
|
+
expect(s.add({ finish_reason: "tool_calls" })).toEqual([{ type: "action.called", callId: "call_2", name: "broken", input: "not json" }]);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it("tool-result 独立到达,配对靠 core 的 deriveRunFacts,这里只管落地成 action.result", () => {
|
|
77
|
+
const s = deltaStream<{ result?: { callId: string; output: string } }>({
|
|
78
|
+
toOps: (f) => (f.result ? [{ kind: "tool-result", callId: f.result.callId, output: f.result.output, status: "completed" }] : []),
|
|
79
|
+
});
|
|
80
|
+
expect(s.add({ result: { callId: "call_1", output: "22C" } })).toEqual([
|
|
81
|
+
{ type: "action.result", callId: "call_1", output: "22C", status: "completed" },
|
|
82
|
+
]);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it("error 操作:落地未完成的文本、置 failed、附一条 error 事件", () => {
|
|
86
|
+
const s = deltaStream({ toOps });
|
|
87
|
+
s.add({ delta: { content: "还没说完" } });
|
|
88
|
+
const events = s.add({ finish_reason: "error" } as never);
|
|
89
|
+
// toOps 不认识 "error" finish_reason,验证一条真正的错误路径
|
|
90
|
+
expect(events).toEqual([]);
|
|
91
|
+
const s2 = deltaStream<{ err?: string }>({ toOps: (f) => (f.err ? [{ kind: "error", message: f.err }] : []) });
|
|
92
|
+
expect(s2.add({ err: "网关超时" })).toEqual([{ type: "error", message: "网关超时" }]);
|
|
93
|
+
expect(s2.failed).toBe(true);
|
|
94
|
+
});
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
// 会话续接(id/capture、history)与 HITL 停轮现场(hold/take)不再是这里的可选件——
|
|
98
|
+
// 它们是 ctx.session(AgentSession)本身的存取器,测试见 src/context/session.test.ts
|
|
99
|
+
// 的 createAgentSession 覆盖(history 新线为空/commit 后可见/线间隔离;capture
|
|
100
|
+
// first-writer-wins/空值忽略;hold/take 一次消费/线间隔离)。
|
|
101
|
+
|
|
102
|
+
describe("driveFrameStream", () => {
|
|
103
|
+
function cursorOf<T>(frames: T[]) {
|
|
104
|
+
let i = 0;
|
|
105
|
+
return { async next() { return i < frames.length ? frames[i++] : null; } };
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
it("逐帧喂 reducer,汇总 events / usage,流结束正常 completed", async () => {
|
|
109
|
+
const frames = [{ text: "a" }, { text: "b" }];
|
|
110
|
+
const reducer = {
|
|
111
|
+
usage: { inputTokens: 1, outputTokens: 2 },
|
|
112
|
+
add: (f: { text: string }) => [{ type: "message" as const, role: "assistant" as const, text: f.text }],
|
|
113
|
+
};
|
|
114
|
+
const turn = await driveFrameStream(cursorOf(frames), reducer, lineCtx());
|
|
115
|
+
expect(turn.status).toBe("completed");
|
|
116
|
+
expect(turn.events).toHaveLength(2);
|
|
117
|
+
expect(turn.usage).toEqual({ inputTokens: 1, outputTokens: 2 });
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it("onFrame 返回 pause:立即停止读流,附加 input.requested,status 置 waiting", async () => {
|
|
121
|
+
let consumed = 0;
|
|
122
|
+
const cursor = {
|
|
123
|
+
async next() {
|
|
124
|
+
consumed++;
|
|
125
|
+
return consumed <= 3 ? { gate: consumed === 2 } : null;
|
|
126
|
+
},
|
|
127
|
+
};
|
|
128
|
+
const reducer = { add: () => [] };
|
|
129
|
+
const turn = await driveFrameStream(cursor, reducer, lineCtx(), (frame) =>
|
|
130
|
+
frame.gate ? { pause: { id: "req1", action: "deploy" } } : undefined,
|
|
131
|
+
);
|
|
132
|
+
expect(turn.status).toBe("waiting");
|
|
133
|
+
expect(consumed).toBe(2); // 第三帧没被读——暂停立即返回,不多读
|
|
134
|
+
expect(turn.events).toEqual([{ type: "input.requested", request: { id: "req1", action: "deploy" } }]);
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
it("onFrame 返回 fail:记一条 error,继续读完,status 置 failed", async () => {
|
|
138
|
+
const frames = [{ err: false }, { err: true }, { err: false }];
|
|
139
|
+
const reducer = { add: () => [] };
|
|
140
|
+
const turn = await driveFrameStream(cursorOf(frames), reducer, lineCtx(), (f) =>
|
|
141
|
+
f.err ? { fail: "网关超时" } : undefined,
|
|
142
|
+
);
|
|
143
|
+
expect(turn.status).toBe("failed");
|
|
144
|
+
expect(turn.events).toEqual([{ type: "error", message: "网关超时" }]);
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
it("reducer.failed 为真时即便没有 onFrame 也判 failed", async () => {
|
|
148
|
+
const reducer = { failed: true, add: () => [] };
|
|
149
|
+
const turn = await driveFrameStream(cursorOf([{}]), reducer, lineCtx());
|
|
150
|
+
expect(turn.status).toBe("failed");
|
|
151
|
+
});
|
|
152
|
+
});
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
// 官方的「拼装方式」件:把一个手写 send 里反复出现的事收成可复用的小东西,
|
|
2
|
+
// 而不是每接一个新后端就重写一遍循环 + Map + if/else。
|
|
3
|
+
//
|
|
4
|
+
// 背景(见 docs-site/zh/guides/write-send.mdx):一轮交互里真正互不相干的事只有几类——
|
|
5
|
+
// 1. 怎么把输入发出去(transport)——真做不掉,adapter 只写这个;
|
|
6
|
+
// 2. 原始数据怎么变成 StreamEvent[]——按数据到达形状分「整段落地」(sdk-streams.ts 的
|
|
7
|
+
// fromXxxEvents,已有)和「逐 token / 逐参数增量」(deltaStream,这里新增)两种官方 reducer;
|
|
8
|
+
// 3. 会话续接与 HITL 暂停恢复——这两件事完全是控制流层面的模式,和任何具体协议无关,
|
|
9
|
+
// 存取器直接挂在 ctx.session 上(history()/id+capture()、hold()/take()),adapter 取用即可,
|
|
10
|
+
// 不需要额外声明什么(见 src/context/session.ts 的 createAgentSession)。
|
|
11
|
+
//
|
|
12
|
+
// driveFrameStream 把「逐帧喂 reducer + 处理特殊传输帧 + 检测 HITL 暂停信号」这个循环收成
|
|
13
|
+
// 一个函数——claude-sdk / codex-sdk / pi-sdk 三个示例里几乎相同的 drainStream,现在只需要
|
|
14
|
+
// 传一个 onFrame 钩子声明「这一帧要不要额外处理」,不用每次重写循环本身。
|
|
15
|
+
|
|
16
|
+
import type { AgentContext, InputRequest, JsonValue, StreamEvent, Turn, Usage } from "../types.ts";
|
|
17
|
+
import type { SseFrameCursor } from "./sdk-streams.ts";
|
|
18
|
+
|
|
19
|
+
// ───────────────────────── driveFrameStream:通用逐帧驱动循环 ─────────────────────────
|
|
20
|
+
|
|
21
|
+
/** sdk-streams.ts 里 ClaudeSdkStream / PiAgentStream / CodexThreadStream 共享的最小形状。 */
|
|
22
|
+
export interface FrameReducer<Frame> {
|
|
23
|
+
add(frame: Frame): StreamEvent[];
|
|
24
|
+
readonly usage?: Usage;
|
|
25
|
+
readonly failed?: boolean;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* 处理一帧「reducer 词汇之外」的事:抓会话 id、识别传输层错误帧、判断是否该在这一帧暂停等人。
|
|
30
|
+
* 返回 `undefined` 什么都不做(reducer 派生的事件已经在 `derived` 里,由 driveFrameStream 收集);
|
|
31
|
+
* 返回 `{ pause }` 立即停止读流、附加一条 `input.requested`、把 status 置为 `"waiting"`;
|
|
32
|
+
* 返回 `{ fail }` 记一条 error 事件,继续读完(有的错误后面还有收尾帧)。
|
|
33
|
+
*/
|
|
34
|
+
export type FrameHook<Frame> = (
|
|
35
|
+
frame: Frame,
|
|
36
|
+
derived: readonly StreamEvent[],
|
|
37
|
+
ctx: AgentContext,
|
|
38
|
+
) => void | { pause: InputRequest } | { fail: string };
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* 逐帧驱动一个 reducer,直到流结束或命中 `onFrame` 的暂停信号。
|
|
42
|
+
* 暂停时不关闭 cursor——调用方多半用 `ctx.session.hold(cursor)` 存住它,回答轮
|
|
43
|
+
* `ctx.session.take()` 取回,下一轮直接接着读同一条流,不重新发起请求。
|
|
44
|
+
*
|
|
45
|
+
* reducer 的帧类型(`RFrame`)允许只覆盖流(`Frame`)的一个子集:adapter 常在 SDK 原生帧之外
|
|
46
|
+
* 混入自己的传输帧(session / server_error…),这些由 `onFrame` 处理,reducer 对认不出的帧
|
|
47
|
+
* 本来就返回 `[]`——所以两个类型参数独立推导,不强求 cursor 和 reducer 同型。
|
|
48
|
+
*/
|
|
49
|
+
export async function driveFrameStream<Frame, RFrame = Frame>(
|
|
50
|
+
cursor: SseFrameCursor<Frame>,
|
|
51
|
+
reducer: FrameReducer<RFrame>,
|
|
52
|
+
ctx: AgentContext,
|
|
53
|
+
onFrame?: FrameHook<Frame>,
|
|
54
|
+
): Promise<Turn> {
|
|
55
|
+
const events: StreamEvent[] = [];
|
|
56
|
+
let transportFailed = false;
|
|
57
|
+
|
|
58
|
+
for (;;) {
|
|
59
|
+
const frame = await cursor.next();
|
|
60
|
+
if (frame === null) break;
|
|
61
|
+
|
|
62
|
+
// 契约:reducer 对不认识的帧返回 [](传输帧走 onFrame),所以这里按 RFrame 喂是安全的。
|
|
63
|
+
const derived = reducer.add(frame as unknown as RFrame);
|
|
64
|
+
events.push(...derived);
|
|
65
|
+
|
|
66
|
+
const outcome = onFrame?.(frame, derived, ctx);
|
|
67
|
+
if (outcome && "pause" in outcome) {
|
|
68
|
+
events.push({ type: "input.requested", request: outcome.pause });
|
|
69
|
+
return { status: "waiting", events, usage: reducer.usage };
|
|
70
|
+
}
|
|
71
|
+
if (outcome && "fail" in outcome) {
|
|
72
|
+
transportFailed = true;
|
|
73
|
+
events.push({ type: "error", message: outcome.fail });
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return { status: transportFailed || reducer.failed ? "failed" : "completed", events, usage: reducer.usage };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// 会话续接(id/capture、history)与 HITL 停轮现场(hold/take)不再是这里的可选「拼装件」——
|
|
81
|
+
// 它们是 ctx.session(AgentSession)本身的存取器,任何 adapter 直接取用,不需要额外声明什么。
|
|
82
|
+
// 见 docs-site/zh/concepts/adapter.mdx 与 src/context/session.ts 的 createAgentSession。
|
|
83
|
+
|
|
84
|
+
// ───────────────────────── deltaStream:逐 token / 逐参数增量累加器 ─────────────────────────
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* 一帧原始数据翻成的操作。多数「返回是流」的后端,拆到最细就是这几种原子操作的排列——
|
|
88
|
+
* 文本要拼接、工具参数要拼接、工具调用与结果要配对、usage 是旁路数字、error 是终态信号。
|
|
89
|
+
* `toOps` 的活只是「这一帧对应哪个/哪些操作」,累加(buffer-by-id、什么时候落地成
|
|
90
|
+
* StreamEvent)由 deltaStream 统一做,不用每个后端各写一遍状态机。
|
|
91
|
+
*/
|
|
92
|
+
export type DeltaOp =
|
|
93
|
+
| { readonly kind: "text-delta"; readonly text: string }
|
|
94
|
+
| { readonly kind: "message-end" }
|
|
95
|
+
| { readonly kind: "thinking-delta"; readonly text: string }
|
|
96
|
+
| { readonly kind: "tool-call-start"; readonly callId: string; readonly name: string }
|
|
97
|
+
| { readonly kind: "tool-args-delta"; readonly callId: string; readonly delta: string }
|
|
98
|
+
/** 该调用的参数已经拼完,落地成 `action.called`(此时通常还没有结果——纯补全式流
|
|
99
|
+
* 一般由调用方在流外真正执行工具;流内也执行的后端随后再喂一条 "tool-result")。 */
|
|
100
|
+
| { readonly kind: "tool-call-end"; readonly callId: string }
|
|
101
|
+
| { readonly kind: "tool-result"; readonly callId: string; readonly output?: JsonValue; readonly status?: "completed" | "failed" | "rejected" }
|
|
102
|
+
| { readonly kind: "usage"; readonly usage: Usage }
|
|
103
|
+
| { readonly kind: "error"; readonly message: string };
|
|
104
|
+
|
|
105
|
+
export interface DeltaStreamSpec<Frame> {
|
|
106
|
+
/** 一帧 → 0~N 个操作(多数协议一帧一个操作;usage 常与收尾帧同帧,可以一次返回两个)。 */
|
|
107
|
+
toOps(frame: Frame): DeltaOp[];
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
interface ToolBuffer {
|
|
111
|
+
name: string;
|
|
112
|
+
args: string;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* 逐 token / 逐参数增量的通用累加器(delta streaming——OpenAI/Anthropic 原始流式 API、
|
|
117
|
+
* 手写的 token-by-token SSE 后端都是这个形状)。文本按 delta 拼接、遇 `message-end` 落地成
|
|
118
|
+
* 一条 `message`;工具参数按 `callId` 拼接、遇 `tool-call-end` 落地成 `action.called`
|
|
119
|
+
* (JSON 解析失败就把拼出来的原始字符串塞进 `input`,不吞错误);`tool-result` 独立到达,
|
|
120
|
+
* 与「整段落地」的 `fromXxxEvents()` 系列同一个 `FrameReducer` 形状,可以直接喂
|
|
121
|
+
* `driveFrameStream`。
|
|
122
|
+
*/
|
|
123
|
+
export function deltaStream<Frame>(spec: DeltaStreamSpec<Frame>): FrameReducer<Frame> {
|
|
124
|
+
let textBuffer = "";
|
|
125
|
+
let thinkingBuffer = "";
|
|
126
|
+
const toolBuffers = new Map<string, ToolBuffer>();
|
|
127
|
+
let usage: Usage | undefined;
|
|
128
|
+
let failed = false;
|
|
129
|
+
|
|
130
|
+
function flushText(events: StreamEvent[]) {
|
|
131
|
+
if (textBuffer) events.push({ type: "message", role: "assistant", text: textBuffer });
|
|
132
|
+
textBuffer = "";
|
|
133
|
+
if (thinkingBuffer) events.push({ type: "thinking", text: thinkingBuffer });
|
|
134
|
+
thinkingBuffer = "";
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
return {
|
|
138
|
+
get usage() {
|
|
139
|
+
return usage;
|
|
140
|
+
},
|
|
141
|
+
get failed() {
|
|
142
|
+
return failed;
|
|
143
|
+
},
|
|
144
|
+
add(frame) {
|
|
145
|
+
const events: StreamEvent[] = [];
|
|
146
|
+
for (const op of spec.toOps(frame)) {
|
|
147
|
+
switch (op.kind) {
|
|
148
|
+
case "text-delta":
|
|
149
|
+
textBuffer += op.text;
|
|
150
|
+
break;
|
|
151
|
+
case "thinking-delta":
|
|
152
|
+
thinkingBuffer += op.text;
|
|
153
|
+
break;
|
|
154
|
+
case "message-end":
|
|
155
|
+
flushText(events);
|
|
156
|
+
break;
|
|
157
|
+
case "tool-call-start":
|
|
158
|
+
toolBuffers.set(op.callId, { name: op.name, args: "" });
|
|
159
|
+
break;
|
|
160
|
+
case "tool-args-delta": {
|
|
161
|
+
const buf = toolBuffers.get(op.callId);
|
|
162
|
+
if (buf) buf.args += op.delta;
|
|
163
|
+
break;
|
|
164
|
+
}
|
|
165
|
+
case "tool-call-end": {
|
|
166
|
+
const buf = toolBuffers.get(op.callId);
|
|
167
|
+
toolBuffers.delete(op.callId);
|
|
168
|
+
let input: JsonValue = null;
|
|
169
|
+
if (buf?.args) {
|
|
170
|
+
try {
|
|
171
|
+
input = JSON.parse(buf.args) as JsonValue;
|
|
172
|
+
} catch {
|
|
173
|
+
input = buf.args; // 拼不出合法 JSON 也别吞掉,原样交给断言/人去看
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
events.push({ type: "action.called", callId: op.callId, name: buf?.name ?? "unknown", input });
|
|
177
|
+
break;
|
|
178
|
+
}
|
|
179
|
+
case "tool-result":
|
|
180
|
+
events.push({ type: "action.result", callId: op.callId, output: op.output, status: op.status ?? "completed" });
|
|
181
|
+
break;
|
|
182
|
+
case "usage":
|
|
183
|
+
usage = op.usage;
|
|
184
|
+
break;
|
|
185
|
+
case "error":
|
|
186
|
+
failed = true;
|
|
187
|
+
flushText(events);
|
|
188
|
+
events.push({ type: "error", message: op.message });
|
|
189
|
+
break;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
return events;
|
|
193
|
+
},
|
|
194
|
+
};
|
|
195
|
+
}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
// agent 域类型:Agent / Adapter 契约、会话与 tracing 导出配置。
|
|
2
|
+
// 「连到哪个被测对象、协议怎么说」的全部契约在这里(见 docs-site/zh/concepts/adapter.mdx)。
|
|
3
|
+
// 能力不再是问卷式声明:t 上解锁什么完全由构造证据决定(见 docs-site 「能力从哪来」一节)。
|
|
4
|
+
|
|
5
|
+
import type { Cleanup } from "../shared/types.ts";
|
|
6
|
+
import type { StreamEvent, TraceSpan, Usage } from "../o11y/types.ts";
|
|
7
|
+
import type { Sandbox } from "../sandbox/types.ts";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* MCP server 描述符 —— Claude Code 与 Codex 共用的扩展插件单元。
|
|
11
|
+
* 在 agent factory config 里声明,setup 阶段写进各自的配置文件。
|
|
12
|
+
*/
|
|
13
|
+
export interface McpServer {
|
|
14
|
+
/** 服务器唯一名(config key)。 */
|
|
15
|
+
name: string;
|
|
16
|
+
/** 启动命令(如 "npx"、"node"、"uvx")。 */
|
|
17
|
+
command: string;
|
|
18
|
+
/** 传给命令的参数。 */
|
|
19
|
+
args?: string[];
|
|
20
|
+
/** 注入服务器进程的环境变量。 */
|
|
21
|
+
env?: Record<string, string>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** 随一轮消息附带的文件(图片等多模态输入)。 */
|
|
25
|
+
export interface InputFile {
|
|
26
|
+
/** 文件名(可选,供 adapter / 模型参考)。 */
|
|
27
|
+
readonly filename?: string;
|
|
28
|
+
/** MIME 类型,如 `image/png`、`image/jpeg`。 */
|
|
29
|
+
readonly mimeType: string;
|
|
30
|
+
/** base64 编码的文件内容(JSON 友好,remote adapter 可直接放进请求体)。 */
|
|
31
|
+
readonly dataBase64: string;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* HITL 回答轮里,人的裁决以结构化形式随 `input.responses` 到达——adapter 不需要解析
|
|
36
|
+
* `text` 去猜哪句回答对应哪个请求、算不算批准。见 docs-site/zh/concepts/adapter.mdx
|
|
37
|
+
* 「不同回答的入参」一节的四种典型形态。
|
|
38
|
+
*/
|
|
39
|
+
export interface InputResponse {
|
|
40
|
+
/** 对应哪条 input.requested 请求;多个请求并停时靠它对位。 */
|
|
41
|
+
readonly requestId: string;
|
|
42
|
+
/** 与 text 二选一:回答命中了请求 options 里的某个 id(approve / deny…)。 */
|
|
43
|
+
readonly optionId?: string;
|
|
44
|
+
/** 与 optionId 二选一:自由文本回答(请求没有选项,或回答不是任何选项)。 */
|
|
45
|
+
readonly text?: string;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface TurnInput {
|
|
49
|
+
readonly text: string;
|
|
50
|
+
/** 本轮附带的文件(图片等)。adapter 自行决定怎么投递;不支持多模态的 adapter 忽略它。 */
|
|
51
|
+
readonly files?: readonly InputFile[];
|
|
52
|
+
/** 仅回答轮(t.respond / t.respondAll):逐请求的结构化回答,按 requestId 对位。 */
|
|
53
|
+
readonly responses?: readonly InputResponse[];
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** adapter 的 send 返回值(事件流为核心)。 */
|
|
57
|
+
export interface Turn {
|
|
58
|
+
readonly events: StreamEvent[];
|
|
59
|
+
readonly data?: unknown;
|
|
60
|
+
readonly status: "completed" | "failed" | "waiting";
|
|
61
|
+
readonly usage?: Usage;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* 本次运行的 OTLP traces 接收信息(仅当配置了 OTel 接入时有——agent 的 `tracing` 块 /
|
|
66
|
+
* config 的 telemetry 存在)。经 ctx.telemetry 交给 agent。span 只进瀑布图,不喂断言。
|
|
67
|
+
* 远程 HTTP 接入的 send 只需要 spread `headers`(每轮一个新 traceparent);接收端点是
|
|
68
|
+
* 启动期配置(`defineConfig({ telemetry: { port } } )`)固定的,不从这里传。
|
|
69
|
+
*/
|
|
70
|
+
export interface Telemetry {
|
|
71
|
+
/** 接收端点(完整路径,形如 http://host.docker.internal:PORT/v1/traces)。 */
|
|
72
|
+
readonly endpoint: string;
|
|
73
|
+
/**
|
|
74
|
+
* env-based 导出的 env(= AgentTracing.env(endpoint) 的结果),ready-to-spread。
|
|
75
|
+
* adapter 的 send 直接 `{ ...ctx.telemetry?.env }` 注入,不必手搓 OTEL_* 拼装。
|
|
76
|
+
*/
|
|
77
|
+
readonly env?: Readonly<Record<string, string>>;
|
|
78
|
+
/**
|
|
79
|
+
* 本轮的 W3C trace context(traceparent),每轮一个新值。send 把它 spread 进 HTTP 请求头
|
|
80
|
+
* (或注入子进程 env)——应用埋点支持 context 传播时,本轮 span 挂到我们给的 trace 下,
|
|
81
|
+
* 并发跑 eval 的 span 归属才精确;不带则回退时间窗口归属(该 agent 自动降为串行)。
|
|
82
|
+
*/
|
|
83
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* agent 的 OTLP 导出配置 —— 「沙箱里怎么让这个 CLI 把 trace 发到 endpoint」。
|
|
88
|
+
* 刻意从 setup 里拆出来:setup 管装 CLI / 写主配置,这里只管 otel 导出。两种投递方式
|
|
89
|
+
* (互不排斥,按 CLI 而定):
|
|
90
|
+
* · env-based(标准 OTEL_* 环境变量,如 bub/Python OTel SDK)—— 用 `env`;
|
|
91
|
+
* · file-based(CLI 自有配置文件,如 codex 的 config.toml [otel] 块)—— 用 `configure`。
|
|
92
|
+
*/
|
|
93
|
+
export interface AgentTracing {
|
|
94
|
+
/**
|
|
95
|
+
* 线协议(codex 发 OTLP/JSON、bub 发 OTLP/protobuf)。接收器按 content-type 自动解码,
|
|
96
|
+
* 此字段仅作声明/日志用,也为将来按协议分流留口。
|
|
97
|
+
*/
|
|
98
|
+
protocol?: "http/json" | "http/protobuf";
|
|
99
|
+
/**
|
|
100
|
+
* 接收器粒度(仅非沙箱 agent;沙箱型每沙箱一个,与此无关):
|
|
101
|
+
* · "attempt"(默认)—— 每个 attempt 一个接收器。适合每轮能切换导出端点的被测对象
|
|
102
|
+
* (进程内 adapter,如内置 aiSdkAgent 的可切换 exporter),attempt 间天然隔离、可全并发。
|
|
103
|
+
* · "run" —— 整个 run 共享一个接收器(每 agent 一个)。**长驻服务必选**:应用的 OTEL_*
|
|
104
|
+
* env 进程启动时读一次,per-attempt 端口会在第一个 attempt 结束时失效。span 逐轮归属
|
|
105
|
+
* (traceparent / 时间窗口 + 串行守卫),见 ctx.telemetry.headers。
|
|
106
|
+
* config 配了 `telemetry`(固定端口,无侵入接入的长驻服务场景)的非沙箱 agent 自动按 "run" 处理。
|
|
107
|
+
*/
|
|
108
|
+
scope?: "attempt" | "run";
|
|
109
|
+
/**
|
|
110
|
+
* env-based 导出:给 endpoint → 返回要注入每轮 send 的 env(纯函数)。运行器把结果
|
|
111
|
+
* 放进 ctx.telemetry.env,send 直接 spread。
|
|
112
|
+
*/
|
|
113
|
+
env?(endpoint: string): Record<string, string>;
|
|
114
|
+
/**
|
|
115
|
+
* file-based 导出:给 sandbox + ctx(ctx.telemetry.endpoint 必有),自己写 / 追加配置文件。
|
|
116
|
+
* 运行器在 agent.setup 之后、首次 send 之前调一次(仅当 tracing 开 + 有 endpoint)。
|
|
117
|
+
* 注:codex 的 [otel.trace_exporter.otlp-http] 是子表,configure 在 setup 写完主配置后
|
|
118
|
+
* 追加到 config.toml 末尾,天然满足「子表在所有上层表之后」。
|
|
119
|
+
*/
|
|
120
|
+
configure?(sandbox: Sandbox, ctx: AgentContext): Promise<void> | void;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* 一条会话线。核心承诺只有一句:**同一条会话线的每次 send 拿到同一个 `ctx.session`,
|
|
125
|
+
* 新会话线(eval 第一轮 / t.newSession() 之后)拿到一个全新的。**
|
|
126
|
+
* 会话续接(`id`/`capture`、`history`)和 HITL 停轮现场(`hold`/`take`)的存取器都在它上面,
|
|
127
|
+
* "第一轮"是新会话线的自然形态,没有要判断的分支;`state` 是这些存取器之外的逃生舱,
|
|
128
|
+
* 框架从不往里写数据。见 docs-site/zh/concepts/adapter.mdx「AgentContext」一节。
|
|
129
|
+
*/
|
|
130
|
+
export interface AgentSession {
|
|
131
|
+
/** 会话续接:服务端记历史。本线记过的会话 id;新会话线是 undefined。 */
|
|
132
|
+
readonly id?: string;
|
|
133
|
+
/** 记回传的会话 id;只在还没记过时落地(first-writer-wins),空值忽略。 */
|
|
134
|
+
capture(id: string | undefined): void;
|
|
135
|
+
/** 会话续接:客户端带全量历史。返回本会话线的历史槽句柄;新线 get() 是空数组。 */
|
|
136
|
+
history<TMsg>(): { get(): TMsg[]; commit(messages: TMsg[]): void };
|
|
137
|
+
/** HITL 停轮现场:存。 */
|
|
138
|
+
hold<T>(state: T): void;
|
|
139
|
+
/** HITL 停轮现场:取,取到即清除(一次消费)。 */
|
|
140
|
+
take<T>(): T | undefined;
|
|
141
|
+
/** 逃生舱:自由状态槽,起始 `{}`,框架从不写入。 */
|
|
142
|
+
readonly state: Record<string, unknown>;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export interface AgentContext {
|
|
146
|
+
readonly signal: AbortSignal;
|
|
147
|
+
readonly model?: string;
|
|
148
|
+
readonly flags: Readonly<Record<string, unknown>>;
|
|
149
|
+
/** 仅沙箱型 agent 有(运行器按 --sandbox 备好)。 */
|
|
150
|
+
readonly sandbox: Sandbox;
|
|
151
|
+
readonly session: AgentSession;
|
|
152
|
+
/**
|
|
153
|
+
* 仅当配置了 OTel 接入时有(agent 的 `tracing` 块 / config 的 telemetry 存在):
|
|
154
|
+
* 本次运行的 OTLP traces 接收信息(endpoint + env-based 导出 env)。
|
|
155
|
+
* 怎么把它交给 CLI 由 agent 的 `tracing` 块声明:env-based 的把 ctx.telemetry.env
|
|
156
|
+
* spread 进 send;file-based 的在 tracing.configure 里写配置。远程 HTTP 接入的 send
|
|
157
|
+
* 只需要把 headers spread 进请求头(每轮一个新 traceparent);端点是启动期配置
|
|
158
|
+
* (defineConfig({ telemetry: { port } }))固定的,不从这里传。
|
|
159
|
+
*/
|
|
160
|
+
readonly telemetry?: Telemetry;
|
|
161
|
+
log(msg: string): void;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* agent 自己的沙箱生命周期(每个沙箱一次,与「每轮 send」分开):
|
|
166
|
+
* `setup` 装 CLI、写配置(model/base/auth 等本轮内不变的东西),`send` 只管把一轮 prompt
|
|
167
|
+
* 跑起来(第一次 fresh / 后续 resume)+ 解析 transcript,`teardown` 清理。
|
|
168
|
+
* 运行器在备好沙箱(上传 / 基线 / eval.setup)后、第一次 send 前调一次 `setup`;
|
|
169
|
+
* `setup` 可返回 cleanup 闭包,与 `teardown` 都在 finally 跑。
|
|
170
|
+
*/
|
|
171
|
+
export type AgentSetup = (sandbox: Sandbox, ctx: AgentContext) => Promise<void | Cleanup> | void | Cleanup;
|
|
172
|
+
export type AgentTeardown = (sandbox: Sandbox, ctx: AgentContext) => Promise<void> | void;
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* 本 agent 的原生 OTLP span → canonical GenAI semconv 的薄 mapper。
|
|
176
|
+
* 由 adapter 声明(和 tracing.env 一样属于「连到谁」的特殊性),core 只调接口——
|
|
177
|
+
* 省略时 core 走通用 heuristic 兜底,不按 agent 名字分支。
|
|
178
|
+
*/
|
|
179
|
+
export type SpanMapper = (spans: TraceSpan[]) => TraceSpan[];
|
|
180
|
+
|
|
181
|
+
/** 注册表里的 agent(defineAgent / defineSandboxAgent 产出)。 */
|
|
182
|
+
export interface Agent {
|
|
183
|
+
readonly name: string;
|
|
184
|
+
/**
|
|
185
|
+
* 内部判别字段(用户不声明):`defineSandboxAgent` 恒设 "sandbox",`defineAgent` 恒设
|
|
186
|
+
* "remote"。t 上的能力不再是问卷式声明,而是构造证据——sandbox 型才解锁
|
|
187
|
+
* `t.sandbox`/`t.fileChanged()` 等文件系统断言,见 docs-site「能力从哪来」一节。
|
|
188
|
+
*/
|
|
189
|
+
readonly kind: "sandbox" | "remote";
|
|
190
|
+
setup?: AgentSetup;
|
|
191
|
+
/** OTLP 导出配置(仅当此字段存在时运行器才为该 agent 开 OTLP 接收);与 setup 分开,见 AgentTracing。 */
|
|
192
|
+
tracing?: AgentTracing;
|
|
193
|
+
/** 原生 span → canonical 的薄 mapper;省略走通用 heuristic。只影响瀑布图。 */
|
|
194
|
+
spanMapper?: SpanMapper;
|
|
195
|
+
send(input: TurnInput, ctx: AgentContext): Promise<Turn>;
|
|
196
|
+
teardown?: AgentTeardown;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
export interface SandboxAgentDef {
|
|
200
|
+
name: string;
|
|
201
|
+
/** 每个沙箱一次:装 CLI、写 config.toml / 鉴权配置。 */
|
|
202
|
+
setup?: AgentSetup;
|
|
203
|
+
/** OTLP 导出配置:沙箱里怎么让 CLI 把 trace 发到 endpoint(env / 配置文件),从 setup 拆出。 */
|
|
204
|
+
tracing?: AgentTracing;
|
|
205
|
+
spanMapper?: SpanMapper;
|
|
206
|
+
/** 每轮一次:跑 prompt(fresh / resume)+ 解析成 events。 */
|
|
207
|
+
send(input: TurnInput, ctx: AgentContext): Promise<Turn>;
|
|
208
|
+
teardown?: AgentTeardown;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export interface RemoteAgentDef {
|
|
212
|
+
name: string;
|
|
213
|
+
setup?: AgentSetup;
|
|
214
|
+
tracing?: AgentTracing;
|
|
215
|
+
spanMapper?: SpanMapper;
|
|
216
|
+
send(input: TurnInput, ctx: AgentContext): Promise<Turn>;
|
|
217
|
+
teardown?: AgentTeardown;
|
|
218
|
+
}
|