niceeval 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/README.zh.md +13 -13
- package/docs/README.md +119 -0
- package/docs/adapters/README.md +60 -0
- package/docs/adapters/authoring.md +179 -0
- package/docs/adapters/coding-agent-skills-plugins.md +324 -0
- package/docs/adapters/collection.md +128 -0
- package/docs/adapters/contract.md +263 -0
- package/docs/adapters/reference/agent-eval.md +215 -0
- package/docs/adapters/reference/agent-loop-apis.md +69 -0
- package/docs/adapters/reference/claude-code-otel-telemetry.md +100 -0
- package/docs/adapters/reference/eve-protocol.md +127 -0
- package/docs/adapters/reference/otel-genai.md +107 -0
- package/docs/adapters/reference/otel-instrumentation.md +65 -0
- package/docs/adapters/targets.md +99 -0
- package/docs/architecture.md +140 -0
- package/docs/assertions.md +388 -0
- package/docs/capabilities-by-construction.md +48 -0
- package/docs/cli.md +171 -0
- package/docs/concepts.md +106 -0
- package/docs/e2e-ci.md +295 -0
- package/docs/eval-authoring.md +319 -0
- package/docs/experiments.md +154 -0
- package/docs/getting-started.md +224 -0
- package/docs/multi-agent.md +145 -0
- package/docs/observability.md +333 -0
- package/docs/origin-integration.md +195 -0
- package/docs/references.md +35 -0
- package/docs/results-format.md +228 -0
- package/docs/runner.md +104 -0
- package/docs/sandbox.md +276 -0
- package/docs/scoring.md +119 -0
- package/docs/source-map.md +106 -0
- package/docs/tier-sync.md +177 -0
- package/docs/view.md +157 -0
- package/docs/vision.md +88 -0
- package/package.json +35 -5
- package/src/agents/ai-sdk-otel.ts +0 -0
- package/src/agents/ai-sdk.test.ts +377 -0
- package/src/agents/ai-sdk.ts +577 -0
- package/src/agents/bub.ts +30 -37
- package/src/agents/claude-code.ts +7 -4
- package/src/agents/codex.ts +15 -10
- package/src/agents/index.ts +43 -1
- package/src/agents/sdk-streams.test.ts +145 -0
- package/src/agents/sdk-streams.ts +457 -0
- package/src/agents/shared.ts +77 -39
- package/src/agents/streaming.test.ts +152 -0
- package/src/agents/streaming.ts +195 -0
- package/src/agents/types.ts +218 -0
- package/src/agents/ui-message-stream.test.ts +249 -0
- package/src/agents/ui-message-stream.ts +372 -0
- package/src/cli.ts +124 -77
- package/src/context/context.test.ts +203 -7
- package/src/context/context.ts +158 -49
- package/src/context/session.test.ts +96 -0
- package/src/context/session.ts +119 -9
- package/src/context/types.ts +205 -0
- package/src/define.ts +4 -14
- package/src/expect/index.ts +8 -71
- package/src/i18n/core.ts +28 -0
- package/src/i18n/en.ts +47 -5
- package/src/i18n/index.ts +5 -17
- package/src/i18n/zh-CN.ts +47 -5
- package/src/index.ts +12 -0
- package/src/loaders/index.ts +8 -39
- package/src/o11y/otlp/canonical.ts +7 -6
- package/src/o11y/otlp/mappers/codex.ts +10 -8
- package/src/o11y/otlp/mappers/index.ts +9 -21
- package/src/o11y/otlp/receiver.ts +25 -7
- package/src/o11y/otlp/sandbox-receiver.ts +57 -21
- package/src/o11y/otlp/turn-otel.test.ts +110 -0
- package/src/o11y/otlp/turn-otel.ts +125 -0
- package/src/o11y/parsers/bub.ts +13 -11
- package/src/o11y/parsers/claude-code.ts +27 -51
- package/src/o11y/parsers/codex.ts +29 -46
- package/src/o11y/parsers/index.ts +5 -14
- package/src/o11y/tool-names.test.ts +61 -0
- package/src/o11y/tool-names.ts +74 -0
- package/src/o11y/types.ts +135 -0
- package/src/runner/attempt.ts +456 -0
- package/src/runner/fingerprint.ts +59 -0
- package/src/runner/remote-sandbox.ts +53 -0
- package/src/runner/report.ts +53 -0
- package/src/runner/reporters/artifacts.ts +25 -4
- package/src/runner/reporters/console.ts +2 -8
- package/src/runner/reporters/live.ts +2 -8
- package/src/runner/reporters/shared.ts +15 -0
- package/src/runner/reporters/table.ts +10 -62
- package/src/runner/run.ts +114 -619
- package/src/runner/types.ts +237 -0
- package/src/sandbox/checkpoint.ts +15 -13
- package/src/sandbox/docker-stream.ts +87 -0
- package/src/sandbox/docker.ts +42 -120
- package/src/sandbox/e2b.ts +24 -75
- package/src/sandbox/local-files.ts +33 -0
- package/src/sandbox/paths.test.ts +85 -0
- package/src/sandbox/paths.ts +45 -0
- package/src/sandbox/resolve.ts +10 -34
- package/src/sandbox/shell.ts +17 -0
- package/src/sandbox/source-files.ts +42 -2
- package/src/sandbox/types.ts +158 -0
- package/src/sandbox/vercel.ts +55 -71
- package/src/scoring/collector.ts +3 -2
- package/src/scoring/judge.ts +72 -146
- package/src/scoring/match.ts +79 -0
- package/src/scoring/types.ts +76 -0
- package/src/shared/aggregate.ts +48 -0
- package/src/shared/format.ts +20 -0
- package/src/shared/outcome.ts +48 -0
- package/src/shared/types.ts +37 -0
- package/src/types.ts +20 -911
- package/src/view/aggregate.ts +103 -0
- package/src/view/app/App.tsx +26 -5
- package/src/view/app/components/AttemptModal.tsx +12 -3
- package/src/view/app/components/CodeView.tsx +9 -5
- package/src/view/app/components/CopyControls.tsx +4 -4
- package/src/view/app/components/ExperimentTable.tsx +5 -5
- package/src/view/app/i18n.ts +31 -7
- package/src/view/app/lib/format.ts +17 -20
- package/src/view/app/lib/outcome.ts +4 -16
- package/src/view/app/main.tsx +3 -5
- package/src/view/app/pages/RunsPage.tsx +2 -2
- package/src/view/app/pages/TracesPage.tsx +2 -2
- package/src/view/app/shared.ts +2 -1
- package/src/view/app/types.ts +6 -43
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +18 -18
- package/src/view/index.ts +24 -401
- package/src/view/loader.ts +234 -0
- package/src/view/server.ts +136 -0
- package/src/view/shared/types.ts +64 -0
- package/src/view/styles.css +60 -9
package/src/context/context.ts
CHANGED
|
@@ -4,13 +4,16 @@
|
|
|
4
4
|
import { readFile } from "node:fs/promises";
|
|
5
5
|
import { basename, extname } from "node:path";
|
|
6
6
|
import { SessionManager, RunSession, lastAssistantText } from "./session.ts";
|
|
7
|
-
import { AssertionCollector } from "../scoring/collector.ts";
|
|
7
|
+
import { AssertionCollector, computePassed } from "../scoring/collector.ts";
|
|
8
|
+
import { deepEqual, validateSchema } from "../scoring/match.ts";
|
|
8
9
|
import type { Spec } from "../scoring/collector.ts";
|
|
9
10
|
import * as Scoped from "../scoring/scoped.ts";
|
|
10
11
|
import { buildJudge } from "../scoring/judge.ts";
|
|
11
12
|
import { EvalSkipped, EvalRequirementFailed, TurnFailed } from "./control-flow.ts";
|
|
12
13
|
import { deriveRunFacts } from "../o11y/derive.ts";
|
|
13
14
|
import { t } from "../i18n/index.ts";
|
|
15
|
+
import { resolveLocalPath } from "../sandbox/paths.ts";
|
|
16
|
+
import { brief } from "../util.ts";
|
|
14
17
|
import type {
|
|
15
18
|
Agent,
|
|
16
19
|
DiffData,
|
|
@@ -18,7 +21,9 @@ import type {
|
|
|
18
21
|
InputFile,
|
|
19
22
|
InputRequest,
|
|
20
23
|
InputRequestFilter,
|
|
24
|
+
InputResponse,
|
|
21
25
|
JudgeConfig,
|
|
26
|
+
RespondAnswer,
|
|
22
27
|
Sandbox,
|
|
23
28
|
SandboxHandle,
|
|
24
29
|
ScoringContext,
|
|
@@ -61,6 +66,23 @@ export interface ContextDeps {
|
|
|
61
66
|
judge: JudgeConfig | undefined;
|
|
62
67
|
/** tracing agent 的 OTLP 端点(运行器起接收器后注入);经 send ctx 透给 adapter。 */
|
|
63
68
|
telemetry?: Telemetry;
|
|
69
|
+
/** 非沙箱 tracing agent 的共享 OTLP 通道(逐轮 span 归属,只进瀑布图)。 */
|
|
70
|
+
otel?: import("../o11y/otlp/turn-otel.ts").AgentOtelChannel;
|
|
71
|
+
/** Eval definition directory; used to resolve host-side relative fixture paths. */
|
|
72
|
+
evalBaseDir?: string;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* 沙箱能力守卫:非沙箱型 agent(kind !== "sandbox")调文件系统类断言就报清晰错误。
|
|
77
|
+
* 这是唯一仍需要构造证据之外强制检查的能力——`t.file`/`t.fileChanged()` 等直接读沙箱
|
|
78
|
+
* 文件系统,没有沙箱就没有东西可读,不报错会静默返回空结果。其余能力(多轮对话、
|
|
79
|
+
* 工具断言……)都不再问卷式声明,由「做没做到」的构造证据决定,见
|
|
80
|
+
* docs-site/zh/concepts/adapter.mdx「能力从哪来」一节。
|
|
81
|
+
*/
|
|
82
|
+
function capabilityGuard(agentName: string, cap: string, method: string): () => never {
|
|
83
|
+
return () => {
|
|
84
|
+
throw new Error(t("context.capabilityMissing", { agent: agentName, cap, method }));
|
|
85
|
+
};
|
|
64
86
|
}
|
|
65
87
|
|
|
66
88
|
export function createEvalContext(deps: ContextDeps): { context: TestContext; state: ContextState } {
|
|
@@ -72,6 +94,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
|
|
|
72
94
|
signal: deps.signal,
|
|
73
95
|
log: deps.log,
|
|
74
96
|
telemetry: deps.telemetry,
|
|
97
|
+
otel: deps.otel,
|
|
75
98
|
});
|
|
76
99
|
const collector = new AssertionCollector();
|
|
77
100
|
const state: ContextState = {
|
|
@@ -85,6 +108,15 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
|
|
|
85
108
|
return value;
|
|
86
109
|
}
|
|
87
110
|
|
|
111
|
+
/** 断言失败时给 view 看的「实际被检查了什么」,而不是重复 matcher 自己的名字。 */
|
|
112
|
+
function previewCheckedValue(value: unknown): string {
|
|
113
|
+
if (value && typeof value === "object" && typeof (value as { path?: unknown }).path === "string") {
|
|
114
|
+
const content = (value as { content?: unknown }).content;
|
|
115
|
+
if (typeof content === "string") return brief(`// ${(value as { path: string }).path}\n${content}`, 4000);
|
|
116
|
+
}
|
|
117
|
+
return brief(value, 4000);
|
|
118
|
+
}
|
|
119
|
+
|
|
88
120
|
const diffView: DiffView = {
|
|
89
121
|
get: (path) => state.late.diff.generatedFiles[path],
|
|
90
122
|
isEmpty: () =>
|
|
@@ -95,6 +127,9 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
|
|
|
95
127
|
};
|
|
96
128
|
|
|
97
129
|
const sandboxHandle: SandboxHandle = {
|
|
130
|
+
get workdir() {
|
|
131
|
+
return deps.sandbox.workdir;
|
|
132
|
+
},
|
|
98
133
|
get sandboxId() {
|
|
99
134
|
return deps.sandbox.sandboxId;
|
|
100
135
|
},
|
|
@@ -108,7 +143,8 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
|
|
|
108
143
|
readSourceFiles: (opts) => deps.sandbox.readSourceFiles(opts),
|
|
109
144
|
writeFiles: (files, targetDir) => deps.sandbox.writeFiles(files, targetDir),
|
|
110
145
|
uploadFiles: (files, targetDir) => deps.sandbox.uploadFiles(files, targetDir),
|
|
111
|
-
uploadDirectory: (localDir, targetDir, opts) =>
|
|
146
|
+
uploadDirectory: (localDir, targetDir, opts) =>
|
|
147
|
+
deps.sandbox.uploadDirectory(resolveLocalPath(deps.evalBaseDir, localDir), targetDir, opts),
|
|
112
148
|
downloadFile: (path) => deps.sandbox.downloadFile(path),
|
|
113
149
|
uploadFile: (path, content) => deps.sandbox.uploadFile(path, content),
|
|
114
150
|
};
|
|
@@ -149,22 +185,34 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
|
|
|
149
185
|
recordScoped(spec, () => session.events, () => session.lastStatus, () => session.usage);
|
|
150
186
|
|
|
151
187
|
const handle: SessionHandle = {
|
|
152
|
-
send: async (text) => makeTurnHandle(await manager.send(session, text), collector, deps),
|
|
188
|
+
send: async (text) => makeTurnHandle(await manager.send(session, text), collector, deps, text),
|
|
153
189
|
sendFile: async (path, text) =>
|
|
154
|
-
makeTurnHandle(await manager.send(session, text ?? "", [await readInputFile(path)]), collector, deps),
|
|
190
|
+
makeTurnHandle(await manager.send(session, text ?? "", [await readInputFile(path)]), collector, deps, text ?? ""),
|
|
155
191
|
requireInputRequest: (filter) => requireInputRequest(session, filter),
|
|
156
|
-
respond: async (...
|
|
157
|
-
if (
|
|
192
|
+
respond: async (...answers) => {
|
|
193
|
+
if (answers.length === 0) throw new Error(t("hitl.respondEmpty"));
|
|
194
|
+
const built = buildRespondInput(session, answers);
|
|
158
195
|
session.pendingInputRequests.length = 0;
|
|
159
|
-
return makeTurnHandle(
|
|
196
|
+
return makeTurnHandle(
|
|
197
|
+
await manager.send(session, built.text, undefined, built.responses),
|
|
198
|
+
collector,
|
|
199
|
+
deps,
|
|
200
|
+
built.text,
|
|
201
|
+
);
|
|
160
202
|
},
|
|
161
203
|
respondAll: async (optionId) => {
|
|
162
204
|
if (session.pendingInputRequests.length === 0) {
|
|
163
|
-
throw new Error(
|
|
205
|
+
throw new Error(t("hitl.respondAllEmpty"));
|
|
164
206
|
}
|
|
165
|
-
const
|
|
207
|
+
const requests = session.pendingInputRequests.slice();
|
|
208
|
+
for (const request of requests) validateOptionId(request, optionId);
|
|
209
|
+
const responses: InputResponse[] = requests.map((request) => ({
|
|
210
|
+
requestId: requireRequestId(request),
|
|
211
|
+
optionId,
|
|
212
|
+
}));
|
|
166
213
|
session.pendingInputRequests.length = 0;
|
|
167
|
-
|
|
214
|
+
const input = requests.map(() => optionId).join("\n");
|
|
215
|
+
return makeTurnHandle(await manager.send(session, input, undefined, responses), collector, deps, input);
|
|
168
216
|
},
|
|
169
217
|
get reply() {
|
|
170
218
|
return session.lastMessage;
|
|
@@ -203,6 +251,22 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
|
|
|
203
251
|
}
|
|
204
252
|
|
|
205
253
|
const primary = makeSessionHandle(manager.primary);
|
|
254
|
+
|
|
255
|
+
// 沙箱能力守卫:非沙箱型 agent(kind !== "sandbox")把文件系统类动作替换成「一调用就报清晰错误」。
|
|
256
|
+
// 其余能力(多轮对话、工具断言……)不再问卷式声明——没接 ctx.session 续接存取器的 agent
|
|
257
|
+
// 每轮各是新对话,没吐 action.* 事件的 agent 上正断言自然不命中,负断言按事件完整性证明
|
|
258
|
+
// 提示可信度,都不需要在这里拦。
|
|
259
|
+
const guards: Record<string, unknown> = {};
|
|
260
|
+
if (deps.agent.kind !== "sandbox") {
|
|
261
|
+
for (const m of ["file", "fileChanged", "fileDeleted", "notInDiff", "noFailedShellCommands"]) {
|
|
262
|
+
guards[m] = capabilityGuard(deps.agent.name, "sandbox", m);
|
|
263
|
+
}
|
|
264
|
+
Object.defineProperty(guards, "sandbox", {
|
|
265
|
+
get: capabilityGuard(deps.agent.name, "sandbox", "sandbox"),
|
|
266
|
+
enumerable: true,
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
|
|
206
270
|
// primary.reply/sessionId/events/usage/judge 是 getter,读的是 manager.primary 的实时状态。
|
|
207
271
|
// 不能 `{ ...primary, ... }` 展开——对象展开会在展开的那一刻把每个 getter 求值成静态值,
|
|
208
272
|
// 之后 t.reply 就永远冻结在「还没 send 过」的初始状态(空字符串)。改用
|
|
@@ -219,18 +283,28 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
|
|
|
219
283
|
throw new EvalSkipped(reason);
|
|
220
284
|
},
|
|
221
285
|
|
|
222
|
-
check: (value: unknown, assertion: ValueAssertion) =>
|
|
223
|
-
|
|
286
|
+
check: (value: unknown, assertion: ValueAssertion) => {
|
|
287
|
+
// evaluate 读 spec 自己的 severity/threshold(而不是捕获记录时的快照):
|
|
288
|
+
// 句柄的 .gate()/.atLeast() 会事后改写 spec,evidence 判定必须和 finalize 同一口径。
|
|
289
|
+
const spec: Spec = {
|
|
224
290
|
name: assertion.name,
|
|
225
291
|
severity: assertion.severity,
|
|
226
292
|
threshold: assertion.threshold,
|
|
227
|
-
evaluate: async (sc) =>
|
|
228
|
-
|
|
293
|
+
evaluate: async (sc) => {
|
|
294
|
+
const resolved = await resolveValue(value, sc);
|
|
295
|
+
const score = await assertion.score(resolved);
|
|
296
|
+
if (computePassed(spec.severity, spec.threshold, score)) return score;
|
|
297
|
+
return { score, evidence: previewCheckedValue(resolved) };
|
|
298
|
+
},
|
|
299
|
+
};
|
|
300
|
+
return collector.record(spec);
|
|
301
|
+
},
|
|
229
302
|
group: <T,>(title: string, fn: () => Promise<T> | T) => collector.withGroup(title, fn),
|
|
230
303
|
require: async (value: unknown, assertion: ValueAssertion) => {
|
|
231
304
|
const v = value instanceof FileRef ? await deps.sandbox.readFile(value.path).catch(() => "") : value;
|
|
232
305
|
const score = await assertion.score(v);
|
|
233
|
-
|
|
306
|
+
// require 恒为硬门槛(不过即中止 eval),判定口径与 finalize 同一份 computePassed。
|
|
307
|
+
const passed = computePassed("gate", assertion.threshold, score);
|
|
234
308
|
collector.record({
|
|
235
309
|
name: assertion.name,
|
|
236
310
|
severity: "gate",
|
|
@@ -253,6 +327,8 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
|
|
|
253
327
|
{
|
|
254
328
|
...Object.getOwnPropertyDescriptors(primary),
|
|
255
329
|
...Object.getOwnPropertyDescriptors(extra),
|
|
330
|
+
// 守卫最后盖上:缺能力的动作被替换成报错闭包。
|
|
331
|
+
...Object.getOwnPropertyDescriptors(guards),
|
|
256
332
|
},
|
|
257
333
|
) as TestContext;
|
|
258
334
|
|
|
@@ -281,7 +357,7 @@ function mimeTypeFor(path: string): string {
|
|
|
281
357
|
}
|
|
282
358
|
}
|
|
283
359
|
|
|
284
|
-
function makeTurnHandle(turn: Turn, collector: AssertionCollector, deps: ContextDeps): TurnHandle {
|
|
360
|
+
function makeTurnHandle(turn: Turn, collector: AssertionCollector, deps: ContextDeps, input: string): TurnHandle {
|
|
285
361
|
const message = lastAssistantText(turn.events) ?? "";
|
|
286
362
|
const facts = deriveRunFacts(turn.events);
|
|
287
363
|
const usage = turn.usage ?? { inputTokens: 0, outputTokens: 0 };
|
|
@@ -327,7 +403,7 @@ function makeTurnHandle(turn: Turn, collector: AssertionCollector, deps: Context
|
|
|
327
403
|
collector.record({
|
|
328
404
|
name: "outputMatches",
|
|
329
405
|
severity: "gate",
|
|
330
|
-
evaluate: () => (validateSchema(turn.data, schema) ? 1 : 0),
|
|
406
|
+
evaluate: async () => ((await validateSchema(turn.data, schema)) ? 1 : 0),
|
|
331
407
|
}),
|
|
332
408
|
messageIncludes: (token) => scoped(Scoped.messageIncludes(token)),
|
|
333
409
|
succeeded: () => scoped(Scoped.succeeded()),
|
|
@@ -345,7 +421,7 @@ function makeTurnHandle(turn: Turn, collector: AssertionCollector, deps: Context
|
|
|
345
421
|
record: (spec) => collector.record(spec),
|
|
346
422
|
judge: deps.judge,
|
|
347
423
|
getOutput: () => message,
|
|
348
|
-
getInput: () =>
|
|
424
|
+
getInput: () => input,
|
|
349
425
|
signal: deps.signal,
|
|
350
426
|
}),
|
|
351
427
|
};
|
|
@@ -367,6 +443,70 @@ function requireInputRequest(session: RunSession, filter?: InputRequestFilter):
|
|
|
367
443
|
return matches[0] as InputRequest;
|
|
368
444
|
}
|
|
369
445
|
|
|
446
|
+
/** InputRequest.id 是 InputResponse.requestId 的唯一来源;adapter 没给稳定 id 就没法对位。 */
|
|
447
|
+
function requireRequestId(request: InputRequest): string {
|
|
448
|
+
if (!request.id) throw new Error(t("hitl.requestMissingId"));
|
|
449
|
+
return request.id;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/** optionId 必须命中 request.options 里的某个 id,写错直接抛,不会静默传给应用。 */
|
|
453
|
+
function validateOptionId(request: InputRequest, optionId: string): void {
|
|
454
|
+
const optionIds = (request.options ?? []).map((o) => o.id);
|
|
455
|
+
if (!optionIds.includes(optionId)) {
|
|
456
|
+
throw new Error(
|
|
457
|
+
t("hitl.invalidOption", {
|
|
458
|
+
optionId,
|
|
459
|
+
requestId: request.id ?? "?",
|
|
460
|
+
options: optionIds.length > 0 ? optionIds.join(" / ") : t("hitl.noOptions"),
|
|
461
|
+
}),
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* t.respond(...) 的每个参数翻成一条 InputResponse + 拼进 input.text 的那一小段文本。
|
|
468
|
+
* 字符串形式只在恰好一条待处理请求时才能自动对位——命中该请求 options 里的某个 id 就是
|
|
469
|
+
* optionId,否则整句落自由文本;多个请求并停时字符串形式无法消歧,直接抛错,要求改用
|
|
470
|
+
* `{ request, optionId }` / `{ request, text }` 对象形式显式指名。
|
|
471
|
+
*/
|
|
472
|
+
function buildRespondInput(
|
|
473
|
+
session: RunSession,
|
|
474
|
+
answers: readonly (string | RespondAnswer)[],
|
|
475
|
+
): { text: string; responses: InputResponse[] } {
|
|
476
|
+
const pieces: string[] = [];
|
|
477
|
+
const responses: InputResponse[] = [];
|
|
478
|
+
for (const answer of answers) {
|
|
479
|
+
if (typeof answer === "string") {
|
|
480
|
+
const resolved = resolveStringAnswer(session, answer);
|
|
481
|
+
pieces.push(answer);
|
|
482
|
+
responses.push(resolved);
|
|
483
|
+
} else {
|
|
484
|
+
const requestId = requireRequestId(answer.request);
|
|
485
|
+
if (answer.optionId !== undefined) {
|
|
486
|
+
validateOptionId(answer.request, answer.optionId);
|
|
487
|
+
pieces.push(answer.optionId);
|
|
488
|
+
responses.push({ requestId, optionId: answer.optionId });
|
|
489
|
+
} else if (answer.text !== undefined) {
|
|
490
|
+
pieces.push(answer.text);
|
|
491
|
+
responses.push({ requestId, text: answer.text });
|
|
492
|
+
} else {
|
|
493
|
+
throw new Error(t("hitl.answerNeedsOptionOrText"));
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
return { text: pieces.join("\n"), responses };
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
function resolveStringAnswer(session: RunSession, raw: string): InputResponse {
|
|
501
|
+
const pending = session.pendingInputRequests;
|
|
502
|
+
if (pending.length === 0) throw new Error(t("hitl.respondAllEmpty"));
|
|
503
|
+
if (pending.length > 1) throw new Error(t("hitl.stringAmbiguous", { count: pending.length }));
|
|
504
|
+
const request = pending[0] as InputRequest;
|
|
505
|
+
const requestId = requireRequestId(request);
|
|
506
|
+
const optionIds = new Set((request.options ?? []).map((o) => o.id));
|
|
507
|
+
return optionIds.has(raw) ? { requestId, optionId: raw } : { requestId, text: raw };
|
|
508
|
+
}
|
|
509
|
+
|
|
370
510
|
function inputRequestMatches(request: InputRequest, filter?: InputRequestFilter): boolean {
|
|
371
511
|
if (!filter) return true;
|
|
372
512
|
if (filter.id !== undefined && !stringMatches(request.id ?? "", filter.id)) return false;
|
|
@@ -394,34 +534,3 @@ function partialObjectMatches(actual: unknown, expected: Record<string, unknown>
|
|
|
394
534
|
return true;
|
|
395
535
|
}
|
|
396
536
|
|
|
397
|
-
function deepEqual(a: unknown, b: unknown): boolean {
|
|
398
|
-
if (a === b) return true;
|
|
399
|
-
if (typeof a !== typeof b || a === null || b === null || typeof a !== "object") return false;
|
|
400
|
-
if (Array.isArray(a) !== Array.isArray(b)) return false;
|
|
401
|
-
const ak = Object.keys(a as object);
|
|
402
|
-
const bk = Object.keys(b as object);
|
|
403
|
-
if (ak.length !== bk.length) return false;
|
|
404
|
-
for (const k of ak) {
|
|
405
|
-
if (!deepEqual((a as Record<string, unknown>)[k], (b as Record<string, unknown>)[k])) return false;
|
|
406
|
-
}
|
|
407
|
-
return true;
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
function validateSchema(value: unknown, schema: unknown): boolean {
|
|
411
|
-
try {
|
|
412
|
-
const std = (schema as { ["~standard"]?: { validate(v: unknown): { issues?: unknown } } })["~standard"];
|
|
413
|
-
if (std && typeof std.validate === "function") {
|
|
414
|
-
const r = std.validate(value) as { issues?: unknown };
|
|
415
|
-
return !(r && r.issues);
|
|
416
|
-
}
|
|
417
|
-
const zodLike = schema as { safeParse?(v: unknown): { success: boolean }; parse?(v: unknown): unknown };
|
|
418
|
-
if (typeof zodLike.safeParse === "function") return zodLike.safeParse(value).success;
|
|
419
|
-
if (typeof zodLike.parse === "function") {
|
|
420
|
-
zodLike.parse(value);
|
|
421
|
-
return true;
|
|
422
|
-
}
|
|
423
|
-
} catch {
|
|
424
|
-
return false;
|
|
425
|
-
}
|
|
426
|
-
return false;
|
|
427
|
-
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
|
|
3
|
+
import { createAgentSession } from "./session.ts";
|
|
4
|
+
|
|
5
|
+
// createAgentSession() 是 ctx.session 的实现——一条会话线的存取器(见
|
|
6
|
+
// docs-site/zh/concepts/adapter.mdx 的 AgentSession 契约)。这里直接测存取器本身;
|
|
7
|
+
// 端到端的「同一条线同一个 ctx.session」由 SessionManager / RunSession 保证。
|
|
8
|
+
|
|
9
|
+
describe("createAgentSession", () => {
|
|
10
|
+
describe("history()", () => {
|
|
11
|
+
it("新线 get() 是空数组;commit 之后同一条线的 get() 能看见", () => {
|
|
12
|
+
const session = createAgentSession();
|
|
13
|
+
const history = session.history<{ role: string; text: string }>();
|
|
14
|
+
expect(history.get()).toEqual([]);
|
|
15
|
+
|
|
16
|
+
history.commit([{ role: "user", text: "hi" }, { role: "assistant", text: "hello" }]);
|
|
17
|
+
expect(history.get()).toEqual([{ role: "user", text: "hi" }, { role: "assistant", text: "hello" }]);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it("不同会话线的历史互相隔离", () => {
|
|
21
|
+
const a = createAgentSession();
|
|
22
|
+
a.history<{ n: number }>().commit([{ n: 1 }]);
|
|
23
|
+
|
|
24
|
+
const b = createAgentSession();
|
|
25
|
+
expect(b.history<{ n: number }>().get()).toEqual([]); // 全新会话线,看不到 a 的历史
|
|
26
|
+
});
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
describe("capture() / id", () => {
|
|
30
|
+
it("新线 id 是 undefined;capture 之后同一条线的下一轮带上", () => {
|
|
31
|
+
const session = createAgentSession();
|
|
32
|
+
expect(session.id).toBeUndefined(); // 第一轮:新会话线的自然结果
|
|
33
|
+
session.capture("sess-1");
|
|
34
|
+
expect(session.id).toBe("sess-1");
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it("first-writer-wins:后续 capture 不覆盖已记录的 id", () => {
|
|
38
|
+
const session = createAgentSession();
|
|
39
|
+
session.capture("sess-new");
|
|
40
|
+
session.capture("sess-forked"); // 后端可能因 fork 换了新 id,不覆盖正在续接的线
|
|
41
|
+
expect(session.id).toBe("sess-new");
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it("空值 / undefined 被忽略,不落地", () => {
|
|
45
|
+
const session = createAgentSession();
|
|
46
|
+
session.capture(undefined);
|
|
47
|
+
session.capture("");
|
|
48
|
+
expect(session.id).toBeUndefined();
|
|
49
|
+
session.capture("sess-1");
|
|
50
|
+
expect(session.id).toBe("sess-1");
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it("不同会话线互相隔离", () => {
|
|
54
|
+
const a = createAgentSession();
|
|
55
|
+
const b = createAgentSession();
|
|
56
|
+
a.capture("sess-a");
|
|
57
|
+
expect(a.id).toBe("sess-a");
|
|
58
|
+
expect(b.id).toBeUndefined(); // b 是新线,看不到 a 的 id
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
describe("hold() / take()", () => {
|
|
63
|
+
it("take() 只消费一次;没有 hold 过就是 undefined", () => {
|
|
64
|
+
const session = createAgentSession();
|
|
65
|
+
expect(session.take<{ toolCallId: string }>()).toBeUndefined();
|
|
66
|
+
session.hold({ toolCallId: "c1" });
|
|
67
|
+
expect(session.take<{ toolCallId: string }>()).toEqual({ toolCallId: "c1" });
|
|
68
|
+
expect(session.take<{ toolCallId: string }>()).toBeUndefined();
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("不要求会话有 id:第一轮就能 hold(服务端无状态的接口也能停轮)", () => {
|
|
72
|
+
const session = createAgentSession();
|
|
73
|
+
session.hold({ x: 1 });
|
|
74
|
+
expect(session.id).toBeUndefined();
|
|
75
|
+
expect(session.take<{ x: number }>()).toEqual({ x: 1 });
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("按会话线隔离,不同线互不干扰", () => {
|
|
79
|
+
const a = createAgentSession();
|
|
80
|
+
const b = createAgentSession();
|
|
81
|
+
a.hold({ v: "A" });
|
|
82
|
+
b.hold({ v: "B" });
|
|
83
|
+
expect(b.take<{ v: string }>()).toEqual({ v: "B" });
|
|
84
|
+
expect(a.take<{ v: string }>()).toEqual({ v: "A" });
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
describe("state", () => {
|
|
89
|
+
it("起始是 {},框架从不写入——可以被 adapter 当逃生舱自由读写", () => {
|
|
90
|
+
const session = createAgentSession();
|
|
91
|
+
expect(session.state).toEqual({});
|
|
92
|
+
(session.state as Record<string, unknown>).foo = "bar";
|
|
93
|
+
expect(session.state.foo).toBe("bar");
|
|
94
|
+
});
|
|
95
|
+
});
|
|
96
|
+
});
|
package/src/context/session.ts
CHANGED
|
@@ -1,14 +1,79 @@
|
|
|
1
1
|
// 会话驱动:把 t.send(text) 翻成 agent.send(input, ctx),在同一沙箱里多轮 resume /
|
|
2
2
|
// newSession,并把每轮的标准事件流与用量累加进整次运行(供作用域断言 / o11y)。
|
|
3
3
|
|
|
4
|
-
import type { Agent, AgentContext, InputFile, InputRequest, Sandbox, StreamEvent, Telemetry, Turn, Usage } from "../types.ts";
|
|
4
|
+
import type { Agent, AgentContext, AgentSession, InputFile, InputRequest, InputResponse, Sandbox, StreamEvent, Telemetry, TraceSpan, Turn, Usage } from "../types.ts";
|
|
5
|
+
import type { AgentOtelChannel } from "../o11y/otlp/turn-otel.ts";
|
|
5
6
|
import { captureLoc } from "../source-loc.ts";
|
|
6
7
|
import { t } from "../i18n/index.ts";
|
|
7
8
|
|
|
8
|
-
/**
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
/**
|
|
10
|
+
* 一条会话线的存取器实现(见 docs-site/zh/concepts/adapter.mdx 的 AgentSession 契约)。
|
|
11
|
+
* 私有槽都关在闭包里——`state` 只归用户,框架内部数据不往里塞。
|
|
12
|
+
*/
|
|
13
|
+
export function createAgentSession(): AgentSession {
|
|
14
|
+
let capturedId: string | undefined;
|
|
15
|
+
let historyLine: unknown[] = [];
|
|
16
|
+
let held: unknown;
|
|
17
|
+
let hasHeld = false;
|
|
18
|
+
const state: Record<string, unknown> = {};
|
|
19
|
+
|
|
20
|
+
return {
|
|
21
|
+
get id() {
|
|
22
|
+
return capturedId;
|
|
23
|
+
},
|
|
24
|
+
capture(id) {
|
|
25
|
+
if (!id || capturedId !== undefined) return; // 空值忽略;first-writer-wins
|
|
26
|
+
capturedId = id;
|
|
27
|
+
},
|
|
28
|
+
history<TMsg>() {
|
|
29
|
+
return {
|
|
30
|
+
get: () => historyLine as TMsg[],
|
|
31
|
+
commit: (messages: TMsg[]) => {
|
|
32
|
+
historyLine = messages;
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
},
|
|
36
|
+
hold(s) {
|
|
37
|
+
held = s;
|
|
38
|
+
hasHeld = true;
|
|
39
|
+
},
|
|
40
|
+
take<T>() {
|
|
41
|
+
if (!hasHeld) return undefined;
|
|
42
|
+
hasHeld = false;
|
|
43
|
+
const v = held as T;
|
|
44
|
+
held = undefined;
|
|
45
|
+
return v;
|
|
46
|
+
},
|
|
47
|
+
state,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* 一条会话线的可变状态。存取器(id/capture/history/hold/take/state)委托给
|
|
53
|
+
* `createAgentSession()`;index/lastMessage/… 是运行器自己的会话簿记,不属于公开契约。
|
|
54
|
+
*/
|
|
55
|
+
export class RunSession implements AgentSession {
|
|
56
|
+
private readonly session = createAgentSession();
|
|
57
|
+
|
|
58
|
+
get id(): string | undefined {
|
|
59
|
+
return this.session.id;
|
|
60
|
+
}
|
|
61
|
+
capture(id: string | undefined): void {
|
|
62
|
+
this.session.capture(id);
|
|
63
|
+
}
|
|
64
|
+
history<TMsg>() {
|
|
65
|
+
return this.session.history<TMsg>();
|
|
66
|
+
}
|
|
67
|
+
hold<T>(s: T): void {
|
|
68
|
+
this.session.hold(s);
|
|
69
|
+
}
|
|
70
|
+
take<T>(): T | undefined {
|
|
71
|
+
return this.session.take<T>();
|
|
72
|
+
}
|
|
73
|
+
get state(): Record<string, unknown> {
|
|
74
|
+
return this.session.state;
|
|
75
|
+
}
|
|
76
|
+
|
|
12
77
|
index = 1;
|
|
13
78
|
lastMessage = "";
|
|
14
79
|
lastInput = "";
|
|
@@ -27,6 +92,8 @@ export interface SessionDeps {
|
|
|
27
92
|
log(msg: string): void;
|
|
28
93
|
/** tracing agent 的 OTLP 端点(经 send ctx 透给 adapter,用于注入导出 env)。 */
|
|
29
94
|
telemetry?: Telemetry;
|
|
95
|
+
/** 非沙箱 tracing agent 的共享 OTLP 通道(runner 从 run 级池取,经它做逐轮 span 归属)。 */
|
|
96
|
+
otel?: AgentOtelChannel;
|
|
30
97
|
}
|
|
31
98
|
|
|
32
99
|
export class SessionManager {
|
|
@@ -35,6 +102,13 @@ export class SessionManager {
|
|
|
35
102
|
readonly usage: Usage = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, requests: 0 };
|
|
36
103
|
lastStatus: "completed" | "failed" | "waiting" = "completed";
|
|
37
104
|
|
|
105
|
+
/** 归属到本 attempt 的 span(逐轮攒;attempt 末尾连同 sweep 的迟到 span 一起挂 trace)。 */
|
|
106
|
+
readonly otelSpans: TraceSpan[] = [];
|
|
107
|
+
/** 本 attempt 各轮的 traceId(attempt 末尾按它 sweep 迟到 span)。 */
|
|
108
|
+
readonly otelTraceIds = new Set<string>();
|
|
109
|
+
private warnedWindowAttribution = false;
|
|
110
|
+
private warnedNoSpans = false;
|
|
111
|
+
|
|
38
112
|
readonly primary: RunSession;
|
|
39
113
|
private readonly sessions: RunSession[] = [];
|
|
40
114
|
private turnCount = 0;
|
|
@@ -51,7 +125,12 @@ export class SessionManager {
|
|
|
51
125
|
return s;
|
|
52
126
|
}
|
|
53
127
|
|
|
54
|
-
async send(
|
|
128
|
+
async send(
|
|
129
|
+
session: RunSession,
|
|
130
|
+
text: string,
|
|
131
|
+
files?: readonly InputFile[],
|
|
132
|
+
responses?: readonly InputResponse[],
|
|
133
|
+
): Promise<Turn> {
|
|
55
134
|
// 抓住作者调 t.send / t.sendFile 那一行(view 把回复叠回这一行)。
|
|
56
135
|
const loc = captureLoc();
|
|
57
136
|
const ctx: AgentContext = {
|
|
@@ -59,7 +138,7 @@ export class SessionManager {
|
|
|
59
138
|
model: this.deps.model,
|
|
60
139
|
flags: this.deps.flags,
|
|
61
140
|
sandbox: this.deps.sandbox,
|
|
62
|
-
session
|
|
141
|
+
session,
|
|
63
142
|
telemetry: this.deps.telemetry,
|
|
64
143
|
log: this.deps.log,
|
|
65
144
|
};
|
|
@@ -78,7 +157,9 @@ export class SessionManager {
|
|
|
78
157
|
this.allEvents.push(userEvent);
|
|
79
158
|
session.events.push(userEvent);
|
|
80
159
|
session.pendingInputRequests.length = 0;
|
|
81
|
-
const turn =
|
|
160
|
+
const turn = this.deps.otel
|
|
161
|
+
? await this.sendWithOtel(this.deps.otel, { text, files, responses }, ctx)
|
|
162
|
+
: await this.deps.agent.send({ text, files, responses }, ctx);
|
|
82
163
|
|
|
83
164
|
this.allEvents.push(...turn.events);
|
|
84
165
|
session.events.push(...turn.events);
|
|
@@ -91,7 +172,6 @@ export class SessionManager {
|
|
|
91
172
|
accumulateUsage(this.usage, turn.usage);
|
|
92
173
|
accumulateUsage(session.usage, turn.usage);
|
|
93
174
|
}
|
|
94
|
-
session.isNew = false;
|
|
95
175
|
session.lastStatus = turn.status;
|
|
96
176
|
this.lastStatus = turn.status;
|
|
97
177
|
const reply = lastAssistantText(turn.events);
|
|
@@ -104,6 +184,36 @@ export class SessionManager {
|
|
|
104
184
|
);
|
|
105
185
|
return turn;
|
|
106
186
|
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* 经共享 OTLP 通道跑一轮:本轮的 traceparent 经 ctx.telemetry.headers 交给 adapter,
|
|
190
|
+
* 返回后按 traceId / 时间窗口把本轮 span 归属进瀑布图。span 只进瀑布图,不进事件流、
|
|
191
|
+
* 不喂断言——断言依据全部来自 send 返回的 Turn。
|
|
192
|
+
*/
|
|
193
|
+
private async sendWithOtel(
|
|
194
|
+
otel: AgentOtelChannel,
|
|
195
|
+
input: { text: string; files?: readonly InputFile[]; responses?: readonly InputResponse[] },
|
|
196
|
+
ctx: AgentContext,
|
|
197
|
+
): Promise<Turn> {
|
|
198
|
+
const r = await otel.runTurn((headers) => {
|
|
199
|
+
const turnCtx: AgentContext = ctx.telemetry
|
|
200
|
+
? { ...ctx, telemetry: { ...ctx.telemetry, headers } }
|
|
201
|
+
: ctx;
|
|
202
|
+
return this.deps.agent.send(input, turnCtx);
|
|
203
|
+
});
|
|
204
|
+
this.otelSpans.push(...r.spans);
|
|
205
|
+
this.otelTraceIds.add(r.traceId);
|
|
206
|
+
|
|
207
|
+
if (r.attribution === "window" && r.spans.length > 0 && !this.warnedWindowAttribution) {
|
|
208
|
+
this.warnedWindowAttribution = true;
|
|
209
|
+
this.deps.log(t("otel.windowAttribution"));
|
|
210
|
+
}
|
|
211
|
+
if (r.spans.length === 0 && !this.warnedNoSpans) {
|
|
212
|
+
this.warnedNoSpans = true;
|
|
213
|
+
this.deps.log(t("otel.noSpans"));
|
|
214
|
+
}
|
|
215
|
+
return r.result;
|
|
216
|
+
}
|
|
107
217
|
}
|
|
108
218
|
|
|
109
219
|
export function lastAssistantText(events: readonly StreamEvent[]): string | undefined {
|