niceeval 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +8 -8
  2. package/README.zh.md +13 -13
  3. package/docs/README.md +119 -0
  4. package/docs/adapters/README.md +60 -0
  5. package/docs/adapters/authoring.md +179 -0
  6. package/docs/adapters/coding-agent-skills-plugins.md +324 -0
  7. package/docs/adapters/collection.md +128 -0
  8. package/docs/adapters/contract.md +263 -0
  9. package/docs/adapters/reference/agent-eval.md +215 -0
  10. package/docs/adapters/reference/agent-loop-apis.md +69 -0
  11. package/docs/adapters/reference/claude-code-otel-telemetry.md +100 -0
  12. package/docs/adapters/reference/eve-protocol.md +127 -0
  13. package/docs/adapters/reference/otel-genai.md +107 -0
  14. package/docs/adapters/reference/otel-instrumentation.md +65 -0
  15. package/docs/adapters/targets.md +99 -0
  16. package/docs/architecture.md +140 -0
  17. package/docs/assertions.md +388 -0
  18. package/docs/capabilities-by-construction.md +48 -0
  19. package/docs/cli.md +171 -0
  20. package/docs/concepts.md +106 -0
  21. package/docs/e2e-ci.md +295 -0
  22. package/docs/eval-authoring.md +319 -0
  23. package/docs/experiments.md +154 -0
  24. package/docs/getting-started.md +224 -0
  25. package/docs/multi-agent.md +145 -0
  26. package/docs/observability.md +333 -0
  27. package/docs/origin-integration.md +195 -0
  28. package/docs/references.md +35 -0
  29. package/docs/results-format.md +228 -0
  30. package/docs/runner.md +104 -0
  31. package/docs/sandbox.md +276 -0
  32. package/docs/scoring.md +119 -0
  33. package/docs/source-map.md +106 -0
  34. package/docs/tier-sync.md +177 -0
  35. package/docs/view.md +157 -0
  36. package/docs/vision.md +88 -0
  37. package/package.json +35 -5
  38. package/src/agents/ai-sdk-otel.ts +0 -0
  39. package/src/agents/ai-sdk.test.ts +377 -0
  40. package/src/agents/ai-sdk.ts +577 -0
  41. package/src/agents/bub.ts +30 -37
  42. package/src/agents/claude-code.ts +7 -4
  43. package/src/agents/codex.ts +15 -10
  44. package/src/agents/index.ts +43 -1
  45. package/src/agents/sdk-streams.test.ts +145 -0
  46. package/src/agents/sdk-streams.ts +457 -0
  47. package/src/agents/shared.ts +77 -39
  48. package/src/agents/streaming.test.ts +152 -0
  49. package/src/agents/streaming.ts +195 -0
  50. package/src/agents/types.ts +218 -0
  51. package/src/agents/ui-message-stream.test.ts +249 -0
  52. package/src/agents/ui-message-stream.ts +372 -0
  53. package/src/cli.ts +124 -77
  54. package/src/context/context.test.ts +203 -7
  55. package/src/context/context.ts +158 -49
  56. package/src/context/session.test.ts +96 -0
  57. package/src/context/session.ts +119 -9
  58. package/src/context/types.ts +205 -0
  59. package/src/define.ts +4 -14
  60. package/src/expect/index.ts +8 -71
  61. package/src/i18n/core.ts +28 -0
  62. package/src/i18n/en.ts +47 -5
  63. package/src/i18n/index.ts +5 -17
  64. package/src/i18n/zh-CN.ts +47 -5
  65. package/src/index.ts +12 -0
  66. package/src/loaders/index.ts +8 -39
  67. package/src/o11y/otlp/canonical.ts +7 -6
  68. package/src/o11y/otlp/mappers/codex.ts +10 -8
  69. package/src/o11y/otlp/mappers/index.ts +9 -21
  70. package/src/o11y/otlp/receiver.ts +25 -7
  71. package/src/o11y/otlp/sandbox-receiver.ts +57 -21
  72. package/src/o11y/otlp/turn-otel.test.ts +110 -0
  73. package/src/o11y/otlp/turn-otel.ts +125 -0
  74. package/src/o11y/parsers/bub.ts +13 -11
  75. package/src/o11y/parsers/claude-code.ts +27 -51
  76. package/src/o11y/parsers/codex.ts +29 -46
  77. package/src/o11y/parsers/index.ts +5 -14
  78. package/src/o11y/tool-names.test.ts +61 -0
  79. package/src/o11y/tool-names.ts +74 -0
  80. package/src/o11y/types.ts +135 -0
  81. package/src/runner/attempt.ts +456 -0
  82. package/src/runner/fingerprint.ts +59 -0
  83. package/src/runner/remote-sandbox.ts +53 -0
  84. package/src/runner/report.ts +53 -0
  85. package/src/runner/reporters/artifacts.ts +25 -4
  86. package/src/runner/reporters/console.ts +2 -8
  87. package/src/runner/reporters/live.ts +2 -8
  88. package/src/runner/reporters/shared.ts +15 -0
  89. package/src/runner/reporters/table.ts +10 -62
  90. package/src/runner/run.ts +114 -619
  91. package/src/runner/types.ts +237 -0
  92. package/src/sandbox/checkpoint.ts +15 -13
  93. package/src/sandbox/docker-stream.ts +87 -0
  94. package/src/sandbox/docker.ts +42 -120
  95. package/src/sandbox/e2b.ts +24 -75
  96. package/src/sandbox/local-files.ts +33 -0
  97. package/src/sandbox/paths.test.ts +85 -0
  98. package/src/sandbox/paths.ts +45 -0
  99. package/src/sandbox/resolve.ts +10 -34
  100. package/src/sandbox/shell.ts +17 -0
  101. package/src/sandbox/source-files.ts +42 -2
  102. package/src/sandbox/types.ts +158 -0
  103. package/src/sandbox/vercel.ts +55 -71
  104. package/src/scoring/collector.ts +3 -2
  105. package/src/scoring/judge.ts +72 -146
  106. package/src/scoring/match.ts +79 -0
  107. package/src/scoring/types.ts +76 -0
  108. package/src/shared/aggregate.ts +48 -0
  109. package/src/shared/format.ts +20 -0
  110. package/src/shared/outcome.ts +48 -0
  111. package/src/shared/types.ts +37 -0
  112. package/src/types.ts +20 -911
  113. package/src/view/aggregate.ts +103 -0
  114. package/src/view/app/App.tsx +26 -5
  115. package/src/view/app/components/AttemptModal.tsx +12 -3
  116. package/src/view/app/components/CodeView.tsx +9 -5
  117. package/src/view/app/components/CopyControls.tsx +4 -4
  118. package/src/view/app/components/ExperimentTable.tsx +5 -5
  119. package/src/view/app/i18n.ts +31 -7
  120. package/src/view/app/lib/format.ts +17 -20
  121. package/src/view/app/lib/outcome.ts +4 -16
  122. package/src/view/app/main.tsx +3 -5
  123. package/src/view/app/pages/RunsPage.tsx +2 -2
  124. package/src/view/app/pages/TracesPage.tsx +2 -2
  125. package/src/view/app/shared.ts +2 -1
  126. package/src/view/app/types.ts +6 -43
  127. package/src/view/client-dist/app.css +1 -1
  128. package/src/view/client-dist/app.js +18 -18
  129. package/src/view/index.ts +24 -401
  130. package/src/view/loader.ts +234 -0
  131. package/src/view/server.ts +136 -0
  132. package/src/view/shared/types.ts +64 -0
  133. package/src/view/styles.css +60 -9
@@ -4,13 +4,16 @@
4
4
  import { readFile } from "node:fs/promises";
5
5
  import { basename, extname } from "node:path";
6
6
  import { SessionManager, RunSession, lastAssistantText } from "./session.ts";
7
- import { AssertionCollector } from "../scoring/collector.ts";
7
+ import { AssertionCollector, computePassed } from "../scoring/collector.ts";
8
+ import { deepEqual, validateSchema } from "../scoring/match.ts";
8
9
  import type { Spec } from "../scoring/collector.ts";
9
10
  import * as Scoped from "../scoring/scoped.ts";
10
11
  import { buildJudge } from "../scoring/judge.ts";
11
12
  import { EvalSkipped, EvalRequirementFailed, TurnFailed } from "./control-flow.ts";
12
13
  import { deriveRunFacts } from "../o11y/derive.ts";
13
14
  import { t } from "../i18n/index.ts";
15
+ import { resolveLocalPath } from "../sandbox/paths.ts";
16
+ import { brief } from "../util.ts";
14
17
  import type {
15
18
  Agent,
16
19
  DiffData,
@@ -18,7 +21,9 @@ import type {
18
21
  InputFile,
19
22
  InputRequest,
20
23
  InputRequestFilter,
24
+ InputResponse,
21
25
  JudgeConfig,
26
+ RespondAnswer,
22
27
  Sandbox,
23
28
  SandboxHandle,
24
29
  ScoringContext,
@@ -61,6 +66,23 @@ export interface ContextDeps {
61
66
  judge: JudgeConfig | undefined;
62
67
  /** tracing agent 的 OTLP 端点(运行器起接收器后注入);经 send ctx 透给 adapter。 */
63
68
  telemetry?: Telemetry;
69
+ /** 非沙箱 tracing agent 的共享 OTLP 通道(逐轮 span 归属,只进瀑布图)。 */
70
+ otel?: import("../o11y/otlp/turn-otel.ts").AgentOtelChannel;
71
+ /** Eval definition directory; used to resolve host-side relative fixture paths. */
72
+ evalBaseDir?: string;
73
+ }
74
+
75
+ /**
76
+ * 沙箱能力守卫:非沙箱型 agent(kind !== "sandbox")调文件系统类断言就报清晰错误。
77
+ * 这是唯一仍需要构造证据之外强制检查的能力——`t.file`/`t.fileChanged()` 等直接读沙箱
78
+ * 文件系统,没有沙箱就没有东西可读,不报错会静默返回空结果。其余能力(多轮对话、
79
+ * 工具断言……)都不再问卷式声明,由「做没做到」的构造证据决定,见
80
+ * docs-site/zh/concepts/adapter.mdx「能力从哪来」一节。
81
+ */
82
+ function capabilityGuard(agentName: string, cap: string, method: string): () => never {
83
+ return () => {
84
+ throw new Error(t("context.capabilityMissing", { agent: agentName, cap, method }));
85
+ };
64
86
  }
65
87
 
66
88
  export function createEvalContext(deps: ContextDeps): { context: TestContext; state: ContextState } {
@@ -72,6 +94,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
72
94
  signal: deps.signal,
73
95
  log: deps.log,
74
96
  telemetry: deps.telemetry,
97
+ otel: deps.otel,
75
98
  });
76
99
  const collector = new AssertionCollector();
77
100
  const state: ContextState = {
@@ -85,6 +108,15 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
85
108
  return value;
86
109
  }
87
110
 
111
+ /** 断言失败时给 view 看的「实际被检查了什么」,而不是重复 matcher 自己的名字。 */
112
+ function previewCheckedValue(value: unknown): string {
113
+ if (value && typeof value === "object" && typeof (value as { path?: unknown }).path === "string") {
114
+ const content = (value as { content?: unknown }).content;
115
+ if (typeof content === "string") return brief(`// ${(value as { path: string }).path}\n${content}`, 4000);
116
+ }
117
+ return brief(value, 4000);
118
+ }
119
+
88
120
  const diffView: DiffView = {
89
121
  get: (path) => state.late.diff.generatedFiles[path],
90
122
  isEmpty: () =>
@@ -95,6 +127,9 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
95
127
  };
96
128
 
97
129
  const sandboxHandle: SandboxHandle = {
130
+ get workdir() {
131
+ return deps.sandbox.workdir;
132
+ },
98
133
  get sandboxId() {
99
134
  return deps.sandbox.sandboxId;
100
135
  },
@@ -108,7 +143,8 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
108
143
  readSourceFiles: (opts) => deps.sandbox.readSourceFiles(opts),
109
144
  writeFiles: (files, targetDir) => deps.sandbox.writeFiles(files, targetDir),
110
145
  uploadFiles: (files, targetDir) => deps.sandbox.uploadFiles(files, targetDir),
111
- uploadDirectory: (localDir, targetDir, opts) => deps.sandbox.uploadDirectory(localDir, targetDir, opts),
146
+ uploadDirectory: (localDir, targetDir, opts) =>
147
+ deps.sandbox.uploadDirectory(resolveLocalPath(deps.evalBaseDir, localDir), targetDir, opts),
112
148
  downloadFile: (path) => deps.sandbox.downloadFile(path),
113
149
  uploadFile: (path, content) => deps.sandbox.uploadFile(path, content),
114
150
  };
@@ -149,22 +185,34 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
149
185
  recordScoped(spec, () => session.events, () => session.lastStatus, () => session.usage);
150
186
 
151
187
  const handle: SessionHandle = {
152
- send: async (text) => makeTurnHandle(await manager.send(session, text), collector, deps),
188
+ send: async (text) => makeTurnHandle(await manager.send(session, text), collector, deps, text),
153
189
  sendFile: async (path, text) =>
154
- makeTurnHandle(await manager.send(session, text ?? "", [await readInputFile(path)]), collector, deps),
190
+ makeTurnHandle(await manager.send(session, text ?? "", [await readInputFile(path)]), collector, deps, text ?? ""),
155
191
  requireInputRequest: (filter) => requireInputRequest(session, filter),
156
- respond: async (...responses) => {
157
- if (responses.length === 0) throw new Error("respond() requires at least one response");
192
+ respond: async (...answers) => {
193
+ if (answers.length === 0) throw new Error(t("hitl.respondEmpty"));
194
+ const built = buildRespondInput(session, answers);
158
195
  session.pendingInputRequests.length = 0;
159
- return makeTurnHandle(await manager.send(session, responses.join("\n")), collector, deps);
196
+ return makeTurnHandle(
197
+ await manager.send(session, built.text, undefined, built.responses),
198
+ collector,
199
+ deps,
200
+ built.text,
201
+ );
160
202
  },
161
203
  respondAll: async (optionId) => {
162
204
  if (session.pendingInputRequests.length === 0) {
163
- throw new Error("respondAll() requires at least one pending input request");
205
+ throw new Error(t("hitl.respondAllEmpty"));
164
206
  }
165
- const responses = session.pendingInputRequests.map(() => optionId);
207
+ const requests = session.pendingInputRequests.slice();
208
+ for (const request of requests) validateOptionId(request, optionId);
209
+ const responses: InputResponse[] = requests.map((request) => ({
210
+ requestId: requireRequestId(request),
211
+ optionId,
212
+ }));
166
213
  session.pendingInputRequests.length = 0;
167
- return makeTurnHandle(await manager.send(session, responses.join("\n")), collector, deps);
214
+ const input = requests.map(() => optionId).join("\n");
215
+ return makeTurnHandle(await manager.send(session, input, undefined, responses), collector, deps, input);
168
216
  },
169
217
  get reply() {
170
218
  return session.lastMessage;
@@ -203,6 +251,22 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
203
251
  }
204
252
 
205
253
  const primary = makeSessionHandle(manager.primary);
254
+
255
+ // 沙箱能力守卫:非沙箱型 agent(kind !== "sandbox")把文件系统类动作替换成「一调用就报清晰错误」。
256
+ // 其余能力(多轮对话、工具断言……)不再问卷式声明——没接 ctx.session 续接存取器的 agent
257
+ // 每轮各是新对话,没吐 action.* 事件的 agent 上正断言自然不命中,负断言按事件完整性证明
258
+ // 提示可信度,都不需要在这里拦。
259
+ const guards: Record<string, unknown> = {};
260
+ if (deps.agent.kind !== "sandbox") {
261
+ for (const m of ["file", "fileChanged", "fileDeleted", "notInDiff", "noFailedShellCommands"]) {
262
+ guards[m] = capabilityGuard(deps.agent.name, "sandbox", m);
263
+ }
264
+ Object.defineProperty(guards, "sandbox", {
265
+ get: capabilityGuard(deps.agent.name, "sandbox", "sandbox"),
266
+ enumerable: true,
267
+ });
268
+ }
269
+
206
270
  // primary.reply/sessionId/events/usage/judge 是 getter,读的是 manager.primary 的实时状态。
207
271
  // 不能 `{ ...primary, ... }` 展开——对象展开会在展开的那一刻把每个 getter 求值成静态值,
208
272
  // 之后 t.reply 就永远冻结在「还没 send 过」的初始状态(空字符串)。改用
@@ -219,18 +283,28 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
219
283
  throw new EvalSkipped(reason);
220
284
  },
221
285
 
222
- check: (value: unknown, assertion: ValueAssertion) =>
223
- collector.record({
286
+ check: (value: unknown, assertion: ValueAssertion) => {
287
+ // evaluate 读 spec 自己的 severity/threshold(而不是捕获记录时的快照):
288
+ // 句柄的 .gate()/.atLeast() 会事后改写 spec,evidence 判定必须和 finalize 同一口径。
289
+ const spec: Spec = {
224
290
  name: assertion.name,
225
291
  severity: assertion.severity,
226
292
  threshold: assertion.threshold,
227
- evaluate: async (sc) => assertion.score(await resolveValue(value, sc)),
228
- }),
293
+ evaluate: async (sc) => {
294
+ const resolved = await resolveValue(value, sc);
295
+ const score = await assertion.score(resolved);
296
+ if (computePassed(spec.severity, spec.threshold, score)) return score;
297
+ return { score, evidence: previewCheckedValue(resolved) };
298
+ },
299
+ };
300
+ return collector.record(spec);
301
+ },
229
302
  group: <T,>(title: string, fn: () => Promise<T> | T) => collector.withGroup(title, fn),
230
303
  require: async (value: unknown, assertion: ValueAssertion) => {
231
304
  const v = value instanceof FileRef ? await deps.sandbox.readFile(value.path).catch(() => "") : value;
232
305
  const score = await assertion.score(v);
233
- const passed = assertion.threshold === undefined ? score > 0 : score >= assertion.threshold;
306
+ // require 恒为硬门槛(不过即中止 eval),判定口径与 finalize 同一份 computePassed。
307
+ const passed = computePassed("gate", assertion.threshold, score);
234
308
  collector.record({
235
309
  name: assertion.name,
236
310
  severity: "gate",
@@ -253,6 +327,8 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
253
327
  {
254
328
  ...Object.getOwnPropertyDescriptors(primary),
255
329
  ...Object.getOwnPropertyDescriptors(extra),
330
+ // 守卫最后盖上:缺能力的动作被替换成报错闭包。
331
+ ...Object.getOwnPropertyDescriptors(guards),
256
332
  },
257
333
  ) as TestContext;
258
334
 
@@ -281,7 +357,7 @@ function mimeTypeFor(path: string): string {
281
357
  }
282
358
  }
283
359
 
284
- function makeTurnHandle(turn: Turn, collector: AssertionCollector, deps: ContextDeps): TurnHandle {
360
+ function makeTurnHandle(turn: Turn, collector: AssertionCollector, deps: ContextDeps, input: string): TurnHandle {
285
361
  const message = lastAssistantText(turn.events) ?? "";
286
362
  const facts = deriveRunFacts(turn.events);
287
363
  const usage = turn.usage ?? { inputTokens: 0, outputTokens: 0 };
@@ -327,7 +403,7 @@ function makeTurnHandle(turn: Turn, collector: AssertionCollector, deps: Context
327
403
  collector.record({
328
404
  name: "outputMatches",
329
405
  severity: "gate",
330
- evaluate: () => (validateSchema(turn.data, schema) ? 1 : 0),
406
+ evaluate: async () => ((await validateSchema(turn.data, schema)) ? 1 : 0),
331
407
  }),
332
408
  messageIncludes: (token) => scoped(Scoped.messageIncludes(token)),
333
409
  succeeded: () => scoped(Scoped.succeeded()),
@@ -345,7 +421,7 @@ function makeTurnHandle(turn: Turn, collector: AssertionCollector, deps: Context
345
421
  record: (spec) => collector.record(spec),
346
422
  judge: deps.judge,
347
423
  getOutput: () => message,
348
- getInput: () => "",
424
+ getInput: () => input,
349
425
  signal: deps.signal,
350
426
  }),
351
427
  };
@@ -367,6 +443,70 @@ function requireInputRequest(session: RunSession, filter?: InputRequestFilter):
367
443
  return matches[0] as InputRequest;
368
444
  }
369
445
 
446
+ /** InputRequest.id 是 InputResponse.requestId 的唯一来源;adapter 没给稳定 id 就没法对位。 */
447
+ function requireRequestId(request: InputRequest): string {
448
+ if (!request.id) throw new Error(t("hitl.requestMissingId"));
449
+ return request.id;
450
+ }
451
+
452
+ /** optionId 必须命中 request.options 里的某个 id,写错直接抛,不会静默传给应用。 */
453
+ function validateOptionId(request: InputRequest, optionId: string): void {
454
+ const optionIds = (request.options ?? []).map((o) => o.id);
455
+ if (!optionIds.includes(optionId)) {
456
+ throw new Error(
457
+ t("hitl.invalidOption", {
458
+ optionId,
459
+ requestId: request.id ?? "?",
460
+ options: optionIds.length > 0 ? optionIds.join(" / ") : t("hitl.noOptions"),
461
+ }),
462
+ );
463
+ }
464
+ }
465
+
466
+ /**
467
+ * t.respond(...) 的每个参数翻成一条 InputResponse + 拼进 input.text 的那一小段文本。
468
+ * 字符串形式只在恰好一条待处理请求时才能自动对位——命中该请求 options 里的某个 id 就是
469
+ * optionId,否则整句落自由文本;多个请求并停时字符串形式无法消歧,直接抛错,要求改用
470
+ * `{ request, optionId }` / `{ request, text }` 对象形式显式指名。
471
+ */
472
+ function buildRespondInput(
473
+ session: RunSession,
474
+ answers: readonly (string | RespondAnswer)[],
475
+ ): { text: string; responses: InputResponse[] } {
476
+ const pieces: string[] = [];
477
+ const responses: InputResponse[] = [];
478
+ for (const answer of answers) {
479
+ if (typeof answer === "string") {
480
+ const resolved = resolveStringAnswer(session, answer);
481
+ pieces.push(answer);
482
+ responses.push(resolved);
483
+ } else {
484
+ const requestId = requireRequestId(answer.request);
485
+ if (answer.optionId !== undefined) {
486
+ validateOptionId(answer.request, answer.optionId);
487
+ pieces.push(answer.optionId);
488
+ responses.push({ requestId, optionId: answer.optionId });
489
+ } else if (answer.text !== undefined) {
490
+ pieces.push(answer.text);
491
+ responses.push({ requestId, text: answer.text });
492
+ } else {
493
+ throw new Error(t("hitl.answerNeedsOptionOrText"));
494
+ }
495
+ }
496
+ }
497
+ return { text: pieces.join("\n"), responses };
498
+ }
499
+
500
+ function resolveStringAnswer(session: RunSession, raw: string): InputResponse {
501
+ const pending = session.pendingInputRequests;
502
+ if (pending.length === 0) throw new Error(t("hitl.respondAllEmpty"));
503
+ if (pending.length > 1) throw new Error(t("hitl.stringAmbiguous", { count: pending.length }));
504
+ const request = pending[0] as InputRequest;
505
+ const requestId = requireRequestId(request);
506
+ const optionIds = new Set((request.options ?? []).map((o) => o.id));
507
+ return optionIds.has(raw) ? { requestId, optionId: raw } : { requestId, text: raw };
508
+ }
509
+
370
510
  function inputRequestMatches(request: InputRequest, filter?: InputRequestFilter): boolean {
371
511
  if (!filter) return true;
372
512
  if (filter.id !== undefined && !stringMatches(request.id ?? "", filter.id)) return false;
@@ -394,34 +534,3 @@ function partialObjectMatches(actual: unknown, expected: Record<string, unknown>
394
534
  return true;
395
535
  }
396
536
 
397
- function deepEqual(a: unknown, b: unknown): boolean {
398
- if (a === b) return true;
399
- if (typeof a !== typeof b || a === null || b === null || typeof a !== "object") return false;
400
- if (Array.isArray(a) !== Array.isArray(b)) return false;
401
- const ak = Object.keys(a as object);
402
- const bk = Object.keys(b as object);
403
- if (ak.length !== bk.length) return false;
404
- for (const k of ak) {
405
- if (!deepEqual((a as Record<string, unknown>)[k], (b as Record<string, unknown>)[k])) return false;
406
- }
407
- return true;
408
- }
409
-
410
- function validateSchema(value: unknown, schema: unknown): boolean {
411
- try {
412
- const std = (schema as { ["~standard"]?: { validate(v: unknown): { issues?: unknown } } })["~standard"];
413
- if (std && typeof std.validate === "function") {
414
- const r = std.validate(value) as { issues?: unknown };
415
- return !(r && r.issues);
416
- }
417
- const zodLike = schema as { safeParse?(v: unknown): { success: boolean }; parse?(v: unknown): unknown };
418
- if (typeof zodLike.safeParse === "function") return zodLike.safeParse(value).success;
419
- if (typeof zodLike.parse === "function") {
420
- zodLike.parse(value);
421
- return true;
422
- }
423
- } catch {
424
- return false;
425
- }
426
- return false;
427
- }
@@ -0,0 +1,96 @@
1
+ import { describe, expect, it } from "vitest";
2
+
3
+ import { createAgentSession } from "./session.ts";
4
+
5
+ // createAgentSession() 是 ctx.session 的实现——一条会话线的存取器(见
6
+ // docs-site/zh/concepts/adapter.mdx 的 AgentSession 契约)。这里直接测存取器本身;
7
+ // 端到端的「同一条线同一个 ctx.session」由 SessionManager / RunSession 保证。
8
+
9
+ describe("createAgentSession", () => {
10
+ describe("history()", () => {
11
+ it("新线 get() 是空数组;commit 之后同一条线的 get() 能看见", () => {
12
+ const session = createAgentSession();
13
+ const history = session.history<{ role: string; text: string }>();
14
+ expect(history.get()).toEqual([]);
15
+
16
+ history.commit([{ role: "user", text: "hi" }, { role: "assistant", text: "hello" }]);
17
+ expect(history.get()).toEqual([{ role: "user", text: "hi" }, { role: "assistant", text: "hello" }]);
18
+ });
19
+
20
+ it("不同会话线的历史互相隔离", () => {
21
+ const a = createAgentSession();
22
+ a.history<{ n: number }>().commit([{ n: 1 }]);
23
+
24
+ const b = createAgentSession();
25
+ expect(b.history<{ n: number }>().get()).toEqual([]); // 全新会话线,看不到 a 的历史
26
+ });
27
+ });
28
+
29
+ describe("capture() / id", () => {
30
+ it("新线 id 是 undefined;capture 之后同一条线的下一轮带上", () => {
31
+ const session = createAgentSession();
32
+ expect(session.id).toBeUndefined(); // 第一轮:新会话线的自然结果
33
+ session.capture("sess-1");
34
+ expect(session.id).toBe("sess-1");
35
+ });
36
+
37
+ it("first-writer-wins:后续 capture 不覆盖已记录的 id", () => {
38
+ const session = createAgentSession();
39
+ session.capture("sess-new");
40
+ session.capture("sess-forked"); // 后端可能因 fork 换了新 id,不覆盖正在续接的线
41
+ expect(session.id).toBe("sess-new");
42
+ });
43
+
44
+ it("空值 / undefined 被忽略,不落地", () => {
45
+ const session = createAgentSession();
46
+ session.capture(undefined);
47
+ session.capture("");
48
+ expect(session.id).toBeUndefined();
49
+ session.capture("sess-1");
50
+ expect(session.id).toBe("sess-1");
51
+ });
52
+
53
+ it("不同会话线互相隔离", () => {
54
+ const a = createAgentSession();
55
+ const b = createAgentSession();
56
+ a.capture("sess-a");
57
+ expect(a.id).toBe("sess-a");
58
+ expect(b.id).toBeUndefined(); // b 是新线,看不到 a 的 id
59
+ });
60
+ });
61
+
62
+ describe("hold() / take()", () => {
63
+ it("take() 只消费一次;没有 hold 过就是 undefined", () => {
64
+ const session = createAgentSession();
65
+ expect(session.take<{ toolCallId: string }>()).toBeUndefined();
66
+ session.hold({ toolCallId: "c1" });
67
+ expect(session.take<{ toolCallId: string }>()).toEqual({ toolCallId: "c1" });
68
+ expect(session.take<{ toolCallId: string }>()).toBeUndefined();
69
+ });
70
+
71
+ it("不要求会话有 id:第一轮就能 hold(服务端无状态的接口也能停轮)", () => {
72
+ const session = createAgentSession();
73
+ session.hold({ x: 1 });
74
+ expect(session.id).toBeUndefined();
75
+ expect(session.take<{ x: number }>()).toEqual({ x: 1 });
76
+ });
77
+
78
+ it("按会话线隔离,不同线互不干扰", () => {
79
+ const a = createAgentSession();
80
+ const b = createAgentSession();
81
+ a.hold({ v: "A" });
82
+ b.hold({ v: "B" });
83
+ expect(b.take<{ v: string }>()).toEqual({ v: "B" });
84
+ expect(a.take<{ v: string }>()).toEqual({ v: "A" });
85
+ });
86
+ });
87
+
88
+ describe("state", () => {
89
+ it("起始是 {},框架从不写入——可以被 adapter 当逃生舱自由读写", () => {
90
+ const session = createAgentSession();
91
+ expect(session.state).toEqual({});
92
+ (session.state as Record<string, unknown>).foo = "bar";
93
+ expect(session.state.foo).toBe("bar");
94
+ });
95
+ });
96
+ });
@@ -1,14 +1,79 @@
1
1
  // 会话驱动:把 t.send(text) 翻成 agent.send(input, ctx),在同一沙箱里多轮 resume /
2
2
  // newSession,并把每轮的标准事件流与用量累加进整次运行(供作用域断言 / o11y)。
3
3
 
4
- import type { Agent, AgentContext, InputFile, InputRequest, Sandbox, StreamEvent, Telemetry, Turn, Usage } from "../types.ts";
4
+ import type { Agent, AgentContext, AgentSession, InputFile, InputRequest, InputResponse, Sandbox, StreamEvent, Telemetry, TraceSpan, Turn, Usage } from "../types.ts";
5
+ import type { AgentOtelChannel } from "../o11y/otlp/turn-otel.ts";
5
6
  import { captureLoc } from "../source-loc.ts";
6
7
  import { t } from "../i18n/index.ts";
7
8
 
8
- /** 一条会话线的可变状态。adapter 读 isNew 决定是否 --resume,写 id 供下轮续接。 */
9
- export class RunSession {
10
- id: string | undefined = undefined;
11
- isNew = true;
9
+ /**
10
+ * 一条会话线的存取器实现(见 docs-site/zh/concepts/adapter.mdx 的 AgentSession 契约)。
11
+ * 私有槽都关在闭包里——`state` 只归用户,框架内部数据不往里塞。
12
+ */
13
+ export function createAgentSession(): AgentSession {
14
+ let capturedId: string | undefined;
15
+ let historyLine: unknown[] = [];
16
+ let held: unknown;
17
+ let hasHeld = false;
18
+ const state: Record<string, unknown> = {};
19
+
20
+ return {
21
+ get id() {
22
+ return capturedId;
23
+ },
24
+ capture(id) {
25
+ if (!id || capturedId !== undefined) return; // 空值忽略;first-writer-wins
26
+ capturedId = id;
27
+ },
28
+ history<TMsg>() {
29
+ return {
30
+ get: () => historyLine as TMsg[],
31
+ commit: (messages: TMsg[]) => {
32
+ historyLine = messages;
33
+ },
34
+ };
35
+ },
36
+ hold(s) {
37
+ held = s;
38
+ hasHeld = true;
39
+ },
40
+ take<T>() {
41
+ if (!hasHeld) return undefined;
42
+ hasHeld = false;
43
+ const v = held as T;
44
+ held = undefined;
45
+ return v;
46
+ },
47
+ state,
48
+ };
49
+ }
50
+
51
+ /**
52
+ * 一条会话线的可变状态。存取器(id/capture/history/hold/take/state)委托给
53
+ * `createAgentSession()`;index/lastMessage/… 是运行器自己的会话簿记,不属于公开契约。
54
+ */
55
+ export class RunSession implements AgentSession {
56
+ private readonly session = createAgentSession();
57
+
58
+ get id(): string | undefined {
59
+ return this.session.id;
60
+ }
61
+ capture(id: string | undefined): void {
62
+ this.session.capture(id);
63
+ }
64
+ history<TMsg>() {
65
+ return this.session.history<TMsg>();
66
+ }
67
+ hold<T>(s: T): void {
68
+ this.session.hold(s);
69
+ }
70
+ take<T>(): T | undefined {
71
+ return this.session.take<T>();
72
+ }
73
+ get state(): Record<string, unknown> {
74
+ return this.session.state;
75
+ }
76
+
12
77
  index = 1;
13
78
  lastMessage = "";
14
79
  lastInput = "";
@@ -27,6 +92,8 @@ export interface SessionDeps {
27
92
  log(msg: string): void;
28
93
  /** tracing agent 的 OTLP 端点(经 send ctx 透给 adapter,用于注入导出 env)。 */
29
94
  telemetry?: Telemetry;
95
+ /** 非沙箱 tracing agent 的共享 OTLP 通道(runner 从 run 级池取,经它做逐轮 span 归属)。 */
96
+ otel?: AgentOtelChannel;
30
97
  }
31
98
 
32
99
  export class SessionManager {
@@ -35,6 +102,13 @@ export class SessionManager {
35
102
  readonly usage: Usage = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, requests: 0 };
36
103
  lastStatus: "completed" | "failed" | "waiting" = "completed";
37
104
 
105
+ /** 归属到本 attempt 的 span(逐轮攒;attempt 末尾连同 sweep 的迟到 span 一起挂 trace)。 */
106
+ readonly otelSpans: TraceSpan[] = [];
107
+ /** 本 attempt 各轮的 traceId(attempt 末尾按它 sweep 迟到 span)。 */
108
+ readonly otelTraceIds = new Set<string>();
109
+ private warnedWindowAttribution = false;
110
+ private warnedNoSpans = false;
111
+
38
112
  readonly primary: RunSession;
39
113
  private readonly sessions: RunSession[] = [];
40
114
  private turnCount = 0;
@@ -51,7 +125,12 @@ export class SessionManager {
51
125
  return s;
52
126
  }
53
127
 
54
- async send(session: RunSession, text: string, files?: readonly InputFile[]): Promise<Turn> {
128
+ async send(
129
+ session: RunSession,
130
+ text: string,
131
+ files?: readonly InputFile[],
132
+ responses?: readonly InputResponse[],
133
+ ): Promise<Turn> {
55
134
  // 抓住作者调 t.send / t.sendFile 那一行(view 把回复叠回这一行)。
56
135
  const loc = captureLoc();
57
136
  const ctx: AgentContext = {
@@ -59,7 +138,7 @@ export class SessionManager {
59
138
  model: this.deps.model,
60
139
  flags: this.deps.flags,
61
140
  sandbox: this.deps.sandbox,
62
- session: session as unknown as AgentContext["session"],
141
+ session,
63
142
  telemetry: this.deps.telemetry,
64
143
  log: this.deps.log,
65
144
  };
@@ -78,7 +157,9 @@ export class SessionManager {
78
157
  this.allEvents.push(userEvent);
79
158
  session.events.push(userEvent);
80
159
  session.pendingInputRequests.length = 0;
81
- const turn = await this.deps.agent.send({ text, files }, ctx);
160
+ const turn = this.deps.otel
161
+ ? await this.sendWithOtel(this.deps.otel, { text, files, responses }, ctx)
162
+ : await this.deps.agent.send({ text, files, responses }, ctx);
82
163
 
83
164
  this.allEvents.push(...turn.events);
84
165
  session.events.push(...turn.events);
@@ -91,7 +172,6 @@ export class SessionManager {
91
172
  accumulateUsage(this.usage, turn.usage);
92
173
  accumulateUsage(session.usage, turn.usage);
93
174
  }
94
- session.isNew = false;
95
175
  session.lastStatus = turn.status;
96
176
  this.lastStatus = turn.status;
97
177
  const reply = lastAssistantText(turn.events);
@@ -104,6 +184,36 @@ export class SessionManager {
104
184
  );
105
185
  return turn;
106
186
  }
187
+
188
+ /**
189
+ * 经共享 OTLP 通道跑一轮:本轮的 traceparent 经 ctx.telemetry.headers 交给 adapter,
190
+ * 返回后按 traceId / 时间窗口把本轮 span 归属进瀑布图。span 只进瀑布图,不进事件流、
191
+ * 不喂断言——断言依据全部来自 send 返回的 Turn。
192
+ */
193
+ private async sendWithOtel(
194
+ otel: AgentOtelChannel,
195
+ input: { text: string; files?: readonly InputFile[]; responses?: readonly InputResponse[] },
196
+ ctx: AgentContext,
197
+ ): Promise<Turn> {
198
+ const r = await otel.runTurn((headers) => {
199
+ const turnCtx: AgentContext = ctx.telemetry
200
+ ? { ...ctx, telemetry: { ...ctx.telemetry, headers } }
201
+ : ctx;
202
+ return this.deps.agent.send(input, turnCtx);
203
+ });
204
+ this.otelSpans.push(...r.spans);
205
+ this.otelTraceIds.add(r.traceId);
206
+
207
+ if (r.attribution === "window" && r.spans.length > 0 && !this.warnedWindowAttribution) {
208
+ this.warnedWindowAttribution = true;
209
+ this.deps.log(t("otel.windowAttribution"));
210
+ }
211
+ if (r.spans.length === 0 && !this.warnedNoSpans) {
212
+ this.warnedNoSpans = true;
213
+ this.deps.log(t("otel.noSpans"));
214
+ }
215
+ return r.result;
216
+ }
107
217
  }
108
218
 
109
219
  export function lastAssistantText(events: readonly StreamEvent[]): string | undefined {