niceeval 0.5.4 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +55 -0
- package/dist/agents/types.d.ts +321 -0
- package/dist/context/types.d.ts +335 -0
- package/dist/i18n/core.d.ts +11 -0
- package/dist/i18n/en.d.ts +181 -0
- package/dist/i18n/index.d.ts +5 -0
- package/dist/i18n/zh-CN.d.ts +183 -0
- package/dist/o11y/otlp/parse.d.ts +3 -0
- package/dist/o11y/otlp/receiver.d.ts +21 -0
- package/dist/o11y/otlp/turn-otel.d.ts +43 -0
- package/dist/o11y/types.d.ts +190 -0
- package/dist/report/aggregate.d.ts +57 -0
- package/dist/report/aggregate.js +212 -0
- package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
- package/dist/report/built-ins/experiment-comparison.js +13 -0
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +2 -0
- package/dist/report/components.d.ts +180 -0
- package/dist/report/components.js +132 -0
- package/dist/report/compute.d.ts +106 -0
- package/dist/report/compute.js +600 -0
- package/dist/report/flag.d.ts +6 -0
- package/dist/report/flag.js +8 -0
- package/dist/report/format.d.ts +26 -0
- package/dist/report/format.js +109 -0
- package/dist/report/index.d.ts +19 -0
- package/dist/report/index.js +30 -0
- package/dist/report/load.d.ts +12 -0
- package/dist/report/load.js +56 -0
- package/dist/report/locale.d.ts +94 -0
- package/dist/report/locale.js +183 -0
- package/dist/report/metrics.d.ts +21 -0
- package/dist/report/metrics.js +115 -0
- package/dist/report/primitives.d.ts +64 -0
- package/dist/report/primitives.js +113 -0
- package/dist/report/react/AttemptList.d.ts +23 -0
- package/dist/report/react/AttemptList.js +19 -0
- package/dist/report/react/DeltaTable.d.ts +8 -0
- package/dist/report/react/DeltaTable.js +24 -0
- package/dist/report/react/EvalList.d.ts +10 -0
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentList.d.ts +11 -0
- package/dist/report/react/ExperimentList.js +67 -0
- package/dist/report/react/GroupSummary.d.ts +8 -0
- package/dist/report/react/GroupSummary.js +8 -0
- package/dist/report/react/MetricBars.d.ts +10 -0
- package/dist/report/react/MetricBars.js +48 -0
- package/dist/report/react/MetricLine.d.ts +9 -0
- package/dist/report/react/MetricLine.js +68 -0
- package/dist/report/react/MetricMatrix.d.ts +10 -0
- package/dist/report/react/MetricMatrix.js +23 -0
- package/dist/report/react/MetricScatter.d.ts +9 -0
- package/dist/report/react/MetricScatter.js +96 -0
- package/dist/report/react/MetricTable.d.ts +11 -0
- package/dist/report/react/MetricTable.js +26 -0
- package/dist/report/react/RunOverview.d.ts +8 -0
- package/dist/report/react/RunOverview.js +12 -0
- package/dist/report/react/Scoreboard.d.ts +8 -0
- package/dist/report/react/Scoreboard.js +26 -0
- package/dist/report/react/cell.d.ts +9 -0
- package/dist/report/react/cell.js +9 -0
- package/dist/report/react/chart-math.d.ts +14 -0
- package/dist/report/react/chart-math.js +68 -0
- package/dist/report/react/colors.d.ts +14 -0
- package/dist/report/react/colors.js +45 -0
- package/dist/report/react/fixtures.d.ts +19 -0
- package/dist/report/react/fixtures.js +398 -0
- package/dist/report/react/format.d.ts +3 -0
- package/dist/report/react/format.js +7 -0
- package/dist/report/react/index.d.ts +6 -0
- package/dist/report/react/index.js +14 -0
- package/dist/report/report.d.ts +24 -0
- package/dist/report/report.js +49 -0
- package/dist/report/text/faces.d.ts +29 -0
- package/dist/report/text/faces.js +541 -0
- package/dist/report/text/layout.d.ts +75 -0
- package/dist/report/text/layout.js +173 -0
- package/dist/report/text/plot.d.ts +37 -0
- package/dist/report/text/plot.js +81 -0
- package/dist/report/text/table.d.ts +8 -0
- package/dist/report/text/table.js +86 -0
- package/dist/report/tree.d.ts +99 -0
- package/dist/report/tree.js +228 -0
- package/dist/report/types.d.ts +408 -0
- package/dist/report/types.js +4 -0
- package/dist/report/web.d.ts +15 -0
- package/dist/report/web.js +37 -0
- package/dist/results/locator.d.ts +75 -0
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +45 -0
- package/dist/results/select.js +230 -0
- package/dist/results/source-hash.d.ts +9 -0
- package/dist/results/types.d.ts +204 -0
- package/dist/runner/eval-source.d.ts +22 -0
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +332 -0
- package/dist/sandbox/docker-stream.d.ts +24 -0
- package/dist/sandbox/docker.d.ts +84 -0
- package/dist/sandbox/e2b.d.ts +31 -0
- package/dist/sandbox/errors.d.ts +4 -0
- package/dist/sandbox/local-files.d.ts +6 -0
- package/dist/sandbox/paths.d.ts +4 -0
- package/dist/sandbox/registry.d.ts +13 -0
- package/dist/sandbox/resolve.d.ts +40 -0
- package/dist/sandbox/retry.d.ts +10 -0
- package/dist/sandbox/shell.d.ts +11 -0
- package/dist/sandbox/source-files.d.ts +13 -0
- package/dist/sandbox/types.d.ts +203 -0
- package/dist/sandbox/vercel.d.ts +37 -0
- package/dist/scoring/types.d.ts +70 -0
- package/dist/shared/aggregate.d.ts +29 -0
- package/dist/shared/aggregate.js +55 -0
- package/dist/shared/types.d.ts +27 -0
- package/dist/shared/verdict.d.ts +27 -0
- package/dist/shared/verdict.js +34 -0
- package/dist/tty-line.d.ts +6 -0
- package/dist/types.d.ts +7 -0
- package/dist/util.d.ts +22 -0
- package/docs-site/zh/concepts/adapter.mdx +7 -7
- package/docs-site/zh/concepts/assert.mdx +19 -18
- package/docs-site/zh/concepts/evals.mdx +12 -11
- package/docs-site/zh/concepts/experiment.mdx +2 -2
- package/docs-site/zh/concepts/judge.mdx +9 -9
- package/docs-site/zh/concepts/overview.mdx +3 -3
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
- package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +237 -78
- package/docs-site/zh/guides/connect-otel.mdx +3 -3
- package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
- package/docs-site/zh/guides/custom-reports.mdx +196 -36
- package/docs-site/zh/guides/experiments.mdx +7 -1
- package/docs-site/zh/guides/fixtures.mdx +4 -2
- package/docs-site/zh/guides/official-adapters.mdx +31 -9
- package/docs-site/zh/guides/publish-report.mdx +4 -4
- package/docs-site/zh/guides/report-components.mdx +194 -55
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +61 -34
- package/docs-site/zh/guides/runner.mdx +2 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +3 -2
- package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +12 -12
- package/docs-site/zh/guides/viewing-results.mdx +200 -79
- package/docs-site/zh/guides/write-experiment.mdx +9 -1
- package/docs-site/zh/guides/write-send.mdx +4 -3
- package/docs-site/zh/index.mdx +4 -3
- package/docs-site/zh/introduction.mdx +2 -1
- package/docs-site/zh/quickstart.mdx +5 -3
- package/docs-site/zh/reference/builtin-agents.mdx +38 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +36 -13
- package/docs-site/zh/reference/define-agent.mdx +15 -13
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/docs-site/zh/reference/events.mdx +9 -3
- package/docs-site/zh/reference/expect.mdx +13 -8
- package/package.json +10 -6
- package/src/agents/bub.ts +129 -42
- package/src/agents/claude-code.test.ts +152 -0
- package/src/agents/claude-code.ts +127 -13
- package/src/agents/codex.test.ts +192 -0
- package/src/agents/codex.ts +151 -12
- package/src/agents/index.ts +9 -3
- package/src/agents/manifest.ts +41 -0
- package/src/agents/plugin-config.test.ts +94 -0
- package/src/agents/shared.ts +33 -2
- package/src/agents/skills.test.ts +214 -0
- package/src/agents/skills.ts +304 -0
- package/src/agents/streaming.ts +5 -5
- package/src/agents/types.ts +59 -3
- package/src/cli.ts +74 -47
- package/src/context/context.test.ts +20 -2
- package/src/context/control-flow.ts +1 -1
- package/src/context/session.test.ts +92 -1
- package/src/context/session.ts +23 -1
- package/src/context/types.ts +4 -4
- package/src/define.test.ts +8 -8
- package/src/define.ts +11 -11
- package/src/expect/index.ts +5 -5
- package/src/i18n/en.ts +33 -15
- package/src/i18n/zh-CN.ts +37 -21
- package/src/index.ts +1 -1
- package/src/o11y/execution-tree.test.ts +452 -0
- package/src/o11y/execution-tree.ts +367 -0
- package/src/o11y/otlp/receiver.ts +1 -1
- package/src/o11y/otlp/select.ts +7 -3
- package/src/o11y/parsers/claude-code.test.ts +175 -0
- package/src/o11y/parsers/claude-code.ts +32 -0
- package/src/o11y/types.ts +6 -1
- package/src/report/aggregate.ts +26 -20
- package/src/report/built-in-user-parity.test.tsx +640 -0
- package/src/report/built-ins/experiment-comparison.tsx +19 -0
- package/src/report/built-ins/index.ts +2 -0
- package/src/report/components.tsx +225 -50
- package/src/report/compute.ts +335 -54
- package/src/report/dual-render.test.tsx +1129 -0
- package/src/report/flag.ts +1 -1
- package/src/report/format.ts +35 -2
- package/src/report/index.ts +47 -14
- package/src/report/load.ts +1 -1
- package/src/report/locale.ts +229 -0
- package/src/report/metrics.ts +39 -15
- package/src/report/primitives.tsx +100 -2
- package/src/report/react/AttemptList.tsx +113 -0
- package/src/report/react/DeltaTable.tsx +9 -6
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentList.tsx +220 -0
- package/src/report/react/GroupSummary.tsx +66 -0
- package/src/report/react/MetricBars.tsx +24 -13
- package/src/report/react/MetricLine.tsx +21 -21
- package/src/report/react/MetricMatrix.tsx +8 -4
- package/src/report/react/MetricScatter.tsx +126 -96
- package/src/report/react/MetricTable.tsx +115 -15
- package/src/report/react/RunOverview.tsx +48 -26
- package/src/report/react/Scoreboard.tsx +15 -7
- package/src/report/react/cell.tsx +18 -9
- package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
- package/src/report/react/colors.ts +24 -15
- package/src/report/react/enhance.js +168 -0
- package/src/report/react/fixtures.ts +169 -34
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +21 -6
- package/src/report/react/render.test.tsx +180 -31
- package/src/report/react/styles.css +432 -213
- package/src/report/report.test.ts +398 -94
- package/src/report/report.ts +29 -11
- package/src/report/text/faces.ts +344 -96
- package/src/report/text/layout.ts +76 -11
- package/src/report/text/table.ts +98 -0
- package/src/report/tree.ts +103 -20
- package/src/report/types.ts +184 -25
- package/src/report/web.ts +46 -19
- package/src/results/annotated-source.test.ts +146 -0
- package/src/results/annotated-source.ts +128 -0
- package/src/results/attempt-evidence.test.ts +158 -0
- package/src/results/attempt-evidence.ts +158 -0
- package/src/results/attempt-source.ts +48 -0
- package/src/results/copy.ts +154 -139
- package/src/results/format.ts +54 -47
- package/src/results/host-equivalence.test.ts +527 -0
- package/src/results/index.ts +56 -14
- package/src/results/locator.test.ts +198 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +392 -175
- package/src/results/results.test.ts +943 -494
- package/src/results/select.ts +129 -23
- package/src/results/skipped-notice.ts +0 -0
- package/src/results/source-hash.ts +28 -0
- package/src/results/types.ts +98 -62
- package/src/results/writer.ts +304 -176
- package/src/runner/attempt.test.ts +170 -0
- package/src/runner/attempt.ts +48 -13
- package/src/runner/discover.test.ts +65 -0
- package/src/runner/discover.ts +6 -2
- package/src/runner/eval-source.test.ts +121 -0
- package/src/runner/eval-source.ts +45 -0
- package/src/runner/fingerprint.ts +54 -1
- package/src/runner/remote-sandbox.ts +1 -1
- package/src/runner/report.test.ts +6 -8
- package/src/runner/report.ts +5 -4
- package/src/runner/reporters/artifacts.ts +27 -41
- package/src/runner/reporters/braintrust.test.ts +3 -3
- package/src/runner/reporters/braintrust.ts +1 -1
- package/src/runner/reporters/console.ts +6 -6
- package/src/runner/reporters/json.ts +4 -4
- package/src/runner/reporters/live.test.ts +56 -0
- package/src/runner/reporters/live.ts +36 -10
- package/src/runner/reporters/quiet.test.ts +66 -0
- package/src/runner/reporters/quiet.ts +49 -0
- package/src/runner/reporters/shared.ts +5 -5
- package/src/runner/reporters/table.ts +17 -17
- package/src/runner/run.ts +61 -102
- package/src/runner/types.ts +43 -32
- package/src/sandbox/checkpoint.ts +1 -1
- package/src/sandbox/docker.ts +15 -2
- package/src/sandbox/e2b.ts +11 -5
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +3 -3
- package/src/sandbox/local-files.ts +1 -1
- package/src/sandbox/registry.ts +6 -5
- package/src/sandbox/resolve.ts +38 -23
- package/src/sandbox/retry.test.ts +68 -0
- package/src/sandbox/retry.ts +44 -0
- package/src/sandbox/types.ts +24 -24
- package/src/sandbox/vercel.ts +17 -3
- package/src/scoring/collector.ts +3 -3
- package/src/scoring/judge.ts +2 -2
- package/src/scoring/scoped.ts +13 -1
- package/src/scoring/types.ts +7 -7
- package/src/scoring/verdict.ts +4 -4
- package/src/shared/aggregate.ts +23 -3
- package/src/shared/types.ts +2 -2
- package/src/shared/{outcome.ts → verdict.ts} +15 -15
- package/src/show/compose.ts +12 -132
- package/src/show/index.ts +119 -95
- package/src/show/render.ts +462 -159
- package/src/show/show.test.ts +471 -253
- package/src/source-loc.ts +1 -1
- package/src/tty-line.ts +32 -0
- package/src/types.ts +2 -2
- package/src/util.test.ts +21 -1
- package/src/util.ts +5 -1
- package/src/view/app/App.tsx +33 -198
- package/src/view/app/components/AttemptModal.tsx +6 -6
- package/src/view/app/components/CodeView.tsx +2 -2
- package/src/view/app/components/CopyControls.tsx +32 -36
- package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
- package/src/view/app/components/ui/badge.tsx +1 -1
- package/src/view/app/components/ui/dialog.tsx +2 -2
- package/src/view/app/i18n.ts +40 -31
- package/src/view/app/lib/artifact-url.ts +1 -1
- package/src/view/app/lib/attempt-route.test.ts +37 -47
- package/src/view/app/lib/attempt-route.ts +26 -26
- package/src/view/app/lib/rows.ts +9 -112
- package/src/view/app/lib/verdict.ts +25 -0
- package/src/view/app/main.tsx +9 -12
- package/src/view/app/pages/RunsPage.tsx +4 -4
- package/src/view/app/pages/TracesPage.tsx +3 -3
- package/src/view/app/shared.ts +0 -1
- package/src/view/app/types.ts +6 -51
- package/src/view/artifact-serving.test.ts +136 -0
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +21 -21
- package/src/view/data.test.ts +214 -143
- package/src/view/data.ts +166 -120
- package/src/view/index.ts +23 -11
- package/src/view/server.ts +52 -17
- package/src/view/shared/types.ts +22 -31
- package/src/view/styles.css +117 -327
- package/src/view/view-report.test.ts +146 -98
- package/src/report/default-report.tsx +0 -222
- package/src/report/dual-face.test.tsx +0 -527
- package/src/report/react/CaseList.tsx +0 -70
- package/src/view/app/components/CostScoreChart.tsx +0 -147
- package/src/view/app/components/ExperimentTable.tsx +0 -284
- package/src/view/app/components/GroupSelector.tsx +0 -61
- package/src/view/app/components/primitives.tsx +0 -43
- package/src/view/app/lib/outcome.ts +0 -82
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
// ExecutionTree:标准事件流骨架 + 可选 OTel span enrichment 合成的统一执行记录
|
|
2
|
+
// (定稿见 docs/observability.md「OTLP traces → 统一瀑布图」、docs/concepts.md「执行树」词条)。
|
|
3
|
+
//
|
|
4
|
+
// 骨架来自 events,永远不因有没有 spans 而变形:message / thinking / skill.loaded /
|
|
5
|
+
// action(action.called + action.result 按 callId 合并成一个节点)/ subagent(同理)/
|
|
6
|
+
// input.requested / compaction / error,每种事件一个节点,顺序 = 事件出现的顺序
|
|
7
|
+
// (合并节点的位置 = 它 called/loaded 那条事件的位置,result/completed 只更新已有节点,
|
|
8
|
+
// 不产生新位置)。没有 spans 时,`span` 字段整体缺失——渲染层据此判断「timing
|
|
9
|
+
// unavailable」,不是 0,也不是从别处反推的估算值。
|
|
10
|
+
//
|
|
11
|
+
// spans 参数假定调用方已经跑过 otlp/select.ts::selectTraceSpans——本函数不做 firehose
|
|
12
|
+
// 过滤,只做「一批语义 span → 骨架节点」的关联,职责单一。关联只走显式 correlation:
|
|
13
|
+
// span.attributes.call_id(现有 mapper 的既有约定,如 claude-code mapper 把
|
|
14
|
+
// tool_use_id 复制过来)或 GenAI 语义约定的 gen_ai.tool.call.id,与 action/subagent/
|
|
15
|
+
// skill.loaded 节点的 callId 精确相等才算数(skill.loaded 的 callId 是可选的——只有原生
|
|
16
|
+
// 协议把 Skill 加载表达成可关联的工具调用时才有,如 Claude Code 的 Skill tool_use;没有
|
|
17
|
+
// 就不参与关联,不影响骨架本身)——从不按 span 名字、文本内容或时间接近度去猜。一个
|
|
18
|
+
// callId 必须唯一对应一条候选 span 才合并;callId 找不到对应骨架节点,或同一个
|
|
19
|
+
// callId 撞了多条候选 span(无法唯一决定该并给哪个节点),这些 span 一律降级成独立
|
|
20
|
+
// 标注的 telemetry-only 节点,绝不強行择一合并。
|
|
21
|
+
//
|
|
22
|
+
// 合并进节点的是【完整 span 对象】(startMs/endMs/parentSpanId/status/attributes 原样
|
|
23
|
+
// 保留供下钻),不重新定义耗时之类的派生字段——耗时是 endMs - startMs,渲染层自己算。
|
|
24
|
+
// action 节点合并时额外吃下 otlp/select.ts::enrichTraceWithIO 想做的事:那个函数把
|
|
25
|
+
// ToolCall.input/output 反向糊到 span.attributes 上的 io.tool/io.input/io.output/io.status
|
|
26
|
+
// 四个键(给独立的 trace.json/瀑布图用),这里在节点内部原地重做一遍同一份 join(同一个
|
|
27
|
+
// IO_MAX 截断预算,ioText 从 otlp/select.ts 直接复用,没有另立口径)——好处不是"action
|
|
28
|
+
// 节点自己没有 input/output"(它本来就有,且是不截断的完整 JsonValue,比 span 视角更权威),
|
|
29
|
+
// 而是让合并出的 span 本身也带一份和旧 enrichTraceWithIO 输出一致的属性,不强迫下游
|
|
30
|
+
// 区分"这份 span 是不是已经被 enrichTraceWithIO 处理过"才能读到入参出参文本。
|
|
31
|
+
// subagent 节点没有 tool-call 形状的 input 字段,不套用这份 io.* 注入。
|
|
32
|
+
//
|
|
33
|
+
// 没有 timingConfidence 字段的设计取舍:otlp/turn-otel.ts 在 traceparent 缺失时会退化成
|
|
34
|
+
// 「按时间窗口把一批 span 划给某一轮 send」的兜底归属(串行守卫下才可靠,见该文件的
|
|
35
|
+
// TurnSpans.attribution),这确实是比「span 精确挂在某个 callId 下」更弱的保证。但这个
|
|
36
|
+
// 弱保证的信息在到达本函数之前就已经丢了——SessionManager 只在运行期 log 一次 warning
|
|
37
|
+
// (`otel.windowAttribution`),从不把 attribution 写回 TraceSpan 本身,所以传进来的
|
|
38
|
+
// spans 天生就分不出「这条是 traceparent 精确挂上的」还是「这条是窗口兜底扫进来的」。
|
|
39
|
+
// 就算这个信息保留了下来,也不该拿它伪造一个「timing 存在但打折扣」的中间态:窗口兜底
|
|
40
|
+
// 归属的是「这批 span 属于哪一轮 send」,本函数关联的是「这条 span 属于哪个 callId 节点」
|
|
41
|
+
// ——两者是正交的轴。一条 span 不管是靠 traceparent 还是窗口兜底进的 trace,它自己的
|
|
42
|
+
// call_id 属性不变,凭 call_id 精确匹配到节点的正确性完全不受轮次归属精度影响(唯一
|
|
43
|
+
// 会被窗口误差污染的是「这条 span 该算哪一轮」这类跨轮聚合,ExecutionTree 不做这个)。
|
|
44
|
+
// 所以诚实的状态只有二元的:call_id 唯一命中 → 有 span(timing 可信);否则 → 没有,
|
|
45
|
+
// 不发明第三档「有 span 但不确定」。
|
|
46
|
+
|
|
47
|
+
import type { InputRequest, JsonValue, StreamEvent, ToolName, TraceSpan } from "../types.ts";
|
|
48
|
+
import { ioText } from "./otlp/select.ts";
|
|
49
|
+
|
|
50
|
+
// ───────────────────────── 节点类型 ─────────────────────────
|
|
51
|
+
|
|
52
|
+
interface ExecutionNodeBase {
|
|
53
|
+
/** 本函数内确定性生成,同一份 (events, spans) 输入永远产出同一批 id(不是全局稳定 id,
|
|
54
|
+
* 不跨调用持久化——目前没有下游需要跨次运行比对同一个节点)。 */
|
|
55
|
+
id: string;
|
|
56
|
+
/** 唯一关联上的 OTel span(供下钻;渲染层用 endMs - startMs 算耗时)。action 节点的
|
|
57
|
+
* span.attributes 额外补了 io.tool/io.input/io.output/io.status(见 withIoAttributes);
|
|
58
|
+
* 其余节点原样保留。缺失 = timing unavailable——要么这次运行没有 OTel 接入,要么有 span
|
|
59
|
+
* 但没能唯一关联到这个节点上;两种情况都不是「假装有耗时」,统一表现为字段不存在。 */
|
|
60
|
+
span?: TraceSpan;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface ExecutionMessageNode extends ExecutionNodeBase {
|
|
64
|
+
kind: "message";
|
|
65
|
+
role: "assistant" | "user";
|
|
66
|
+
text: string;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export interface ExecutionThinkingNode extends ExecutionNodeBase {
|
|
70
|
+
kind: "thinking";
|
|
71
|
+
text: string;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Skill 加载节点——一等,直接来自 StreamEvent 的 "skill.loaded",不靠工具名/文本猜。 */
|
|
75
|
+
export interface ExecutionSkillNode extends ExecutionNodeBase {
|
|
76
|
+
kind: "skill.loaded";
|
|
77
|
+
skill: string;
|
|
78
|
+
/**
|
|
79
|
+
* 仅当原生协议把 Skill 加载表达成可关联的工具调用时才有,和 StreamEvent 同名字段同一含义。
|
|
80
|
+
* 存在时参与和 action/subagent 节点同一套 callId 关联(见主函数 nodeByCallId):Claude Code
|
|
81
|
+
* 的 Skill 调用本身就是一次 tool_use,OTel mapper 同样会把它的 tool_use_id 复制成
|
|
82
|
+
* span.attributes.call_id,唯一命中时这个节点也应该拿到 timing,不因为节点 kind 是
|
|
83
|
+
* skill.loaded 就被排除在 enrichment 之外。
|
|
84
|
+
*/
|
|
85
|
+
callId?: string;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* action.called + action.result 按 callId 合并成一个节点。`status` 多了 "pending"
|
|
90
|
+
* (StreamEvent 的 action.result.status 没有这一档)——这是 ExecutionTree 独有的中间态,
|
|
91
|
+
* 表示这次运行结束时结果始终没有回来(如超时截断的 transcript),诚实地区分「还没完成」
|
|
92
|
+
* 和「跑完但失败/被拒绝」。
|
|
93
|
+
*/
|
|
94
|
+
export interface ExecutionActionNode extends ExecutionNodeBase {
|
|
95
|
+
kind: "action";
|
|
96
|
+
callId: string;
|
|
97
|
+
/** 原始工具名(未归一化),对齐 StreamEvent.action.called.name。 */
|
|
98
|
+
name: string;
|
|
99
|
+
/** 归一化后的规范工具名,省略表示 adapter 没能归一(等同 ToolCall.name 缺省为 "unknown")。 */
|
|
100
|
+
tool?: ToolName;
|
|
101
|
+
input: JsonValue;
|
|
102
|
+
output?: JsonValue;
|
|
103
|
+
status: "pending" | "completed" | "failed" | "rejected";
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** subagent.called + subagent.completed 按 callId 合并;pending 语义同 ExecutionActionNode。 */
|
|
107
|
+
export interface ExecutionSubagentNode extends ExecutionNodeBase {
|
|
108
|
+
kind: "subagent";
|
|
109
|
+
callId: string;
|
|
110
|
+
name: string;
|
|
111
|
+
remoteUrl?: string;
|
|
112
|
+
output?: JsonValue;
|
|
113
|
+
status: "pending" | "completed" | "failed";
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export interface ExecutionInputRequestedNode extends ExecutionNodeBase {
|
|
117
|
+
kind: "input.requested";
|
|
118
|
+
request: InputRequest;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export interface ExecutionCompactionNode extends ExecutionNodeBase {
|
|
122
|
+
kind: "compaction";
|
|
123
|
+
reason?: string;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export interface ExecutionErrorNode extends ExecutionNodeBase {
|
|
127
|
+
kind: "error";
|
|
128
|
+
message: string;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* span 存在、有意义,但唯一关联不到任何骨架节点——原样保留成独立节点,清楚标注「只有遥测、
|
|
133
|
+
* 没有对应事件」,不悄悄猜着并进某个骨架节点。`span` 是必填字段:这类节点的全部内容就是
|
|
134
|
+
* 这一条 span 本身,没有骨架事件可以叠加。
|
|
135
|
+
*/
|
|
136
|
+
export interface ExecutionTelemetryNode {
|
|
137
|
+
kind: "telemetry";
|
|
138
|
+
id: string;
|
|
139
|
+
span: TraceSpan;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export type ExecutionNode =
|
|
143
|
+
| ExecutionMessageNode
|
|
144
|
+
| ExecutionThinkingNode
|
|
145
|
+
| ExecutionSkillNode
|
|
146
|
+
| ExecutionActionNode
|
|
147
|
+
| ExecutionSubagentNode
|
|
148
|
+
| ExecutionInputRequestedNode
|
|
149
|
+
| ExecutionCompactionNode
|
|
150
|
+
| ExecutionErrorNode
|
|
151
|
+
| ExecutionTelemetryNode;
|
|
152
|
+
|
|
153
|
+
export interface ExecutionTree {
|
|
154
|
+
/**
|
|
155
|
+
* 骨架节点在前,顺序 = 事件出现顺序;telemetry-only 节点(未能唯一关联的 span)
|
|
156
|
+
* 按 span.startMs 追加在骨架之后——它们不属于骨架,不改变骨架的节点/顺序/内容,
|
|
157
|
+
* 只是叠加在末尾的额外证据。
|
|
158
|
+
*/
|
|
159
|
+
nodes: ExecutionNode[];
|
|
160
|
+
/**
|
|
161
|
+
* 这次运行是否提供过任何 span——不代表关联成功,只代表「OTel 接入过」。供渲染层区分
|
|
162
|
+
* 两种不同的诚实提示:整体没有 OTel 接入(该字段为 false,所有节点自然都没有 span),
|
|
163
|
+
* 和 OTel 接入了但这一个节点恰好关联不上(该字段为 true,该节点的 span 仍缺失)。
|
|
164
|
+
*/
|
|
165
|
+
timingAvailable: boolean;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// ───────────────────────── 关联 key ─────────────────────────
|
|
169
|
+
|
|
170
|
+
/** span 的显式 correlation key,按优先级尝试(见模块头注:只认这两个,不猜)。 */
|
|
171
|
+
const CALL_ID_ATTRS = ["call_id", "gen_ai.tool.call.id"] as const;
|
|
172
|
+
|
|
173
|
+
function spanCallId(span: TraceSpan): string | undefined {
|
|
174
|
+
const attrs = span.attributes ?? {};
|
|
175
|
+
for (const key of CALL_ID_ATTRS) {
|
|
176
|
+
const v = attrs[key];
|
|
177
|
+
if (typeof v === "string" && v) return v;
|
|
178
|
+
}
|
|
179
|
+
return undefined;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* 与 `otlp/select.ts` 的 `enrichTraceWithIO` 同口径地给合并上的 span 补
|
|
184
|
+
* `io.tool` / `io.input` / `io.output` / `io.status`——node 的 input/output 已经是
|
|
185
|
+
* 事件流自身的完整 JsonValue(不截断,给程序化消费方);这里另外派生一份文本、按
|
|
186
|
+
* `IO_MAX` 截断,给任何只认 span.attributes 通用形状的下游(比如把 span 原样转存/
|
|
187
|
+
* 展示的调试视图)一份和旧 enrichTraceWithIO 输出完全一致的键。不修改传入的 span 对象。
|
|
188
|
+
*/
|
|
189
|
+
function withIoAttributes(span: TraceSpan, node: ExecutionActionNode): TraceSpan {
|
|
190
|
+
const attributes: Record<string, JsonValue> = { ...span.attributes };
|
|
191
|
+
if (node.name) attributes["io.tool"] = node.name;
|
|
192
|
+
if (node.input !== undefined && node.input !== null) attributes["io.input"] = ioText(node.input);
|
|
193
|
+
if (node.output !== undefined && node.output !== null) attributes["io.output"] = ioText(node.output);
|
|
194
|
+
if (node.status !== "pending") attributes["io.status"] = node.status;
|
|
195
|
+
return { ...span, attributes };
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// ───────────────────────── 主函数 ─────────────────────────
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* 纯函数:把标准事件流(骨架)与一批已挑选好的 OTel span(可选 enrichment)合成一棵
|
|
202
|
+
* ExecutionTree。events 决定节点的存在、顺序与内容;spans 只能给已存在的节点补时间,
|
|
203
|
+
* 从不新增、删除或重排骨架节点。
|
|
204
|
+
*/
|
|
205
|
+
export function buildExecutionTree(events: readonly StreamEvent[], spans: readonly TraceSpan[]): ExecutionTree {
|
|
206
|
+
const nodes: ExecutionNode[] = [];
|
|
207
|
+
const actionById = new Map<string, ExecutionActionNode>();
|
|
208
|
+
const subagentById = new Map<string, ExecutionSubagentNode>();
|
|
209
|
+
const skillByCallId = new Map<string, ExecutionSkillNode>();
|
|
210
|
+
let seq = 0;
|
|
211
|
+
const nextId = (prefix: string): string => `${prefix}-${seq++}`;
|
|
212
|
+
|
|
213
|
+
for (const ev of events) {
|
|
214
|
+
switch (ev.type) {
|
|
215
|
+
case "message":
|
|
216
|
+
nodes.push({ id: nextId("message"), kind: "message", role: ev.role, text: ev.text });
|
|
217
|
+
break;
|
|
218
|
+
|
|
219
|
+
case "thinking":
|
|
220
|
+
nodes.push({ id: nextId("thinking"), kind: "thinking", text: ev.text });
|
|
221
|
+
break;
|
|
222
|
+
|
|
223
|
+
case "skill.loaded": {
|
|
224
|
+
const node: ExecutionSkillNode = { id: nextId("skill"), kind: "skill.loaded", skill: ev.skill, callId: ev.callId };
|
|
225
|
+
if (ev.callId) skillByCallId.set(ev.callId, node);
|
|
226
|
+
nodes.push(node);
|
|
227
|
+
break;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
case "action.called": {
|
|
231
|
+
const node: ExecutionActionNode = {
|
|
232
|
+
id: `action-${ev.callId}`,
|
|
233
|
+
kind: "action",
|
|
234
|
+
callId: ev.callId,
|
|
235
|
+
name: ev.name,
|
|
236
|
+
tool: ev.tool,
|
|
237
|
+
input: ev.input,
|
|
238
|
+
status: "pending",
|
|
239
|
+
};
|
|
240
|
+
actionById.set(ev.callId, node);
|
|
241
|
+
nodes.push(node);
|
|
242
|
+
break;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
case "action.result": {
|
|
246
|
+
const existing = actionById.get(ev.callId);
|
|
247
|
+
if (existing) {
|
|
248
|
+
existing.output = ev.output;
|
|
249
|
+
existing.status = ev.status;
|
|
250
|
+
} else {
|
|
251
|
+
// 只有结果、没配上调用(乱序/截断的 transcript):补一条占位节点,位置 = 结果到达时,
|
|
252
|
+
// 与 derive.ts::deriveRunFacts 的同名兜底一致,不静默丢弃这条结果。
|
|
253
|
+
const node: ExecutionActionNode = {
|
|
254
|
+
id: `action-${ev.callId}`,
|
|
255
|
+
kind: "action",
|
|
256
|
+
callId: ev.callId,
|
|
257
|
+
name: "unknown",
|
|
258
|
+
input: null,
|
|
259
|
+
output: ev.output,
|
|
260
|
+
status: ev.status,
|
|
261
|
+
};
|
|
262
|
+
actionById.set(ev.callId, node);
|
|
263
|
+
nodes.push(node);
|
|
264
|
+
}
|
|
265
|
+
break;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
case "subagent.called": {
|
|
269
|
+
const node: ExecutionSubagentNode = {
|
|
270
|
+
id: `subagent-${ev.callId}`,
|
|
271
|
+
kind: "subagent",
|
|
272
|
+
callId: ev.callId,
|
|
273
|
+
name: ev.name,
|
|
274
|
+
remoteUrl: ev.remoteUrl,
|
|
275
|
+
status: "pending",
|
|
276
|
+
};
|
|
277
|
+
subagentById.set(ev.callId, node);
|
|
278
|
+
nodes.push(node);
|
|
279
|
+
break;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
case "subagent.completed": {
|
|
283
|
+
const existing = subagentById.get(ev.callId);
|
|
284
|
+
if (existing) {
|
|
285
|
+
existing.output = ev.output;
|
|
286
|
+
existing.status = ev.status;
|
|
287
|
+
} else {
|
|
288
|
+
const node: ExecutionSubagentNode = {
|
|
289
|
+
id: `subagent-${ev.callId}`,
|
|
290
|
+
kind: "subagent",
|
|
291
|
+
callId: ev.callId,
|
|
292
|
+
name: "unknown",
|
|
293
|
+
output: ev.output,
|
|
294
|
+
status: ev.status,
|
|
295
|
+
};
|
|
296
|
+
subagentById.set(ev.callId, node);
|
|
297
|
+
nodes.push(node);
|
|
298
|
+
}
|
|
299
|
+
break;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
case "input.requested":
|
|
303
|
+
nodes.push({ id: nextId("input"), kind: "input.requested", request: ev.request });
|
|
304
|
+
break;
|
|
305
|
+
|
|
306
|
+
case "compaction":
|
|
307
|
+
nodes.push({ id: nextId("compaction"), kind: "compaction", reason: ev.reason });
|
|
308
|
+
break;
|
|
309
|
+
|
|
310
|
+
case "error":
|
|
311
|
+
nodes.push({ id: nextId("error"), kind: "error", message: ev.message });
|
|
312
|
+
break;
|
|
313
|
+
|
|
314
|
+
default: {
|
|
315
|
+
// 穷尽性检查:StreamEvent 加新变体时这里编译报错,提醒同步补一个节点类型。
|
|
316
|
+
const _exhaustive: never = ev;
|
|
317
|
+
void _exhaustive;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// callId → 骨架节点(action、subagent、skill.loaded 共用同一命名空间;call id 在一次 run
|
|
323
|
+
// 内按现有约定唯一——deriveRunFacts 等既有折叠逻辑同样这么假设)。skill.loaded 的 callId
|
|
324
|
+
// 是可选的(不是所有原生协议都把 Skill 加载表达成可关联的工具调用),只有存在时才参与。
|
|
325
|
+
const nodeByCallId = new Map<string, ExecutionActionNode | ExecutionSubagentNode | ExecutionSkillNode>();
|
|
326
|
+
for (const n of actionById.values()) nodeByCallId.set(n.callId, n);
|
|
327
|
+
for (const n of subagentById.values()) nodeByCallId.set(n.callId, n);
|
|
328
|
+
for (const n of skillByCallId.values()) nodeByCallId.set(n.callId!, n);
|
|
329
|
+
|
|
330
|
+
// 按 callId 分组候选 span;没有 callId 的直接进「待定为 telemetry-only」。
|
|
331
|
+
const spansByCallId = new Map<string, TraceSpan[]>();
|
|
332
|
+
const uncorrelated: TraceSpan[] = [];
|
|
333
|
+
for (const span of spans) {
|
|
334
|
+
const callId = spanCallId(span);
|
|
335
|
+
if (!callId) {
|
|
336
|
+
uncorrelated.push(span);
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
const list = spansByCallId.get(callId);
|
|
340
|
+
if (list) list.push(span);
|
|
341
|
+
else spansByCallId.set(callId, [span]);
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
const telemetry: ExecutionTelemetryNode[] = [];
|
|
345
|
+
for (const [callId, candidates] of spansByCallId) {
|
|
346
|
+
const node = nodeByCallId.get(callId);
|
|
347
|
+
if (node && candidates.length === 1) {
|
|
348
|
+
const matched = candidates[0];
|
|
349
|
+
// action 节点额外补 io.tool/io.input/io.output/io.status——同 otlp/select.ts 的
|
|
350
|
+
// enrichTraceWithIO 一个口径(同一个 IO_MAX 截断预算),这里在节点内部原地做一遍,
|
|
351
|
+
// 不依赖调用方传入的 spans 是不是已经被 enrichTraceWithIO 处理过。subagent 与
|
|
352
|
+
// skill.loaded 节点没有对应的 tool-call 形状入参/出参字段,原样保留 span,不发明 io.* 键。
|
|
353
|
+
node.span = node.kind === "action" ? withIoAttributes(matched, node) : matched;
|
|
354
|
+
} else {
|
|
355
|
+
// 没有对应骨架节点,或同一 callId 撞了多条 span(无法唯一决定该并给谁)——
|
|
356
|
+
// 都降级为 telemetry-only,不強行择一合并。
|
|
357
|
+
for (const span of candidates) telemetry.push({ id: `telemetry-${span.spanId}`, kind: "telemetry", span });
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
for (const span of uncorrelated) telemetry.push({ id: `telemetry-${span.spanId}`, kind: "telemetry", span });
|
|
361
|
+
telemetry.sort((a, b) => a.span.startMs - b.span.startMs);
|
|
362
|
+
|
|
363
|
+
return {
|
|
364
|
+
nodes: [...nodes, ...telemetry],
|
|
365
|
+
timingAvailable: spans.length > 0,
|
|
366
|
+
};
|
|
367
|
+
}
|
|
@@ -10,7 +10,7 @@ import { parseOtlpTraces } from "./parse.ts";
|
|
|
10
10
|
import { t } from "../../i18n/index.ts";
|
|
11
11
|
|
|
12
12
|
export interface TraceReceiver {
|
|
13
|
-
/** agent 应导出到的完整端点(host
|
|
13
|
+
/** agent 应导出到的完整端点(host 由 provider 定:docker → host.docker.internal)。 */
|
|
14
14
|
endpoint(host: string): string;
|
|
15
15
|
/** 目前为止收到并解析出的全部 span(副本)。 */
|
|
16
16
|
collect(): TraceSpan[];
|
package/src/o11y/otlp/select.ts
CHANGED
|
@@ -48,10 +48,14 @@ export function selectTraceSpans(spans: TraceSpan[]): TraceSpan[] {
|
|
|
48
48
|
return kept.sort((a, b) => a.startMs - b.startMs);
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
/**
|
|
52
|
-
|
|
51
|
+
/**
|
|
52
|
+
* I/O 文本上限:文件内容/命令输出可能很大,截一下别把 trace 撑爆。导出给
|
|
53
|
+
* `execution-tree.ts` 复用——ExecutionTree 的 action 节点给合并上去的 span 补同一份
|
|
54
|
+
* io.* attributes,必须和这里同一个截断预算,不能另立一个数字。
|
|
55
|
+
*/
|
|
56
|
+
export const IO_MAX = 4000;
|
|
53
57
|
|
|
54
|
-
function ioText(v: unknown): string {
|
|
58
|
+
export function ioText(v: unknown): string {
|
|
55
59
|
const s = typeof v === "string" ? v : JSON.stringify(v);
|
|
56
60
|
if (s === undefined) return "";
|
|
57
61
|
return s.length > IO_MAX ? s.slice(0, IO_MAX) + `…(+${s.length - IO_MAX})` : s;
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
// claude-code transcript 解析器单测。
|
|
2
|
+
//
|
|
3
|
+
// 重点覆盖 Skill 加载识别(定稿见 docs/observability.md「OTLP traces → 统一瀑布图」、
|
|
4
|
+
// docs/concepts.md「标准事件流」词条:skill.loaded 是一等事件,不靠工具名/文本猜):
|
|
5
|
+
// 真实原生格式是 tool_use 块 name:"Skill"、input:{ skill:"<skill 名>", args?:"..." }
|
|
6
|
+
// (核对过 Claude Code CLI 自带的 SkillTool 定义,见 claude-code.ts 的 extractSkillName 头注)。
|
|
7
|
+
// 这里额外覆盖大小写、限定名、和「普通工具调用完全不受影响」的回归锁——skill.loaded 是
|
|
8
|
+
// 严格新增,不能改变任何非 Skill tool_use 的既有行为。
|
|
9
|
+
|
|
10
|
+
import { describe, expect, it } from "vitest";
|
|
11
|
+
import { parseClaudeCodeTranscript } from "./claude-code.ts";
|
|
12
|
+
|
|
13
|
+
const line = (obj: object): string => JSON.stringify(obj);
|
|
14
|
+
|
|
15
|
+
function toolUseLine(block: { id: string; name: string; input: object }) {
|
|
16
|
+
return line({ type: "assistant", message: { content: [{ type: "tool_use", ...block }] } });
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function toolResultLine(toolUseId: string, content: unknown, isError = false) {
|
|
20
|
+
return line({ type: "user", message: { content: [{ type: "tool_result", tool_use_id: toolUseId, content, is_error: isError }] } });
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
describe("parseClaudeCodeTranscript — Skill 加载", () => {
|
|
24
|
+
it("Skill tool_use 产出 skill.loaded(带 callId),不产出 action.called", () => {
|
|
25
|
+
const raw = [
|
|
26
|
+
toolUseLine({ id: "toolu_01", name: "Skill", input: { skill: "pdf" } }),
|
|
27
|
+
toolResultLine("toolu_01", "PDF Processing\n\n## Quick start\n..."),
|
|
28
|
+
].join("\n");
|
|
29
|
+
|
|
30
|
+
const { events } = parseClaudeCodeTranscript(raw);
|
|
31
|
+
|
|
32
|
+
expect(events).toEqual([{ type: "skill.loaded", skill: "pdf", callId: "toolu_01" }]);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it("识别限定名(namespace:skill),原样作为 skill 字段透传", () => {
|
|
36
|
+
const raw = toolUseLine({ id: "toolu_02", name: "Skill", input: { skill: "ms-office-suite:pdf" } });
|
|
37
|
+
|
|
38
|
+
const { events } = parseClaudeCodeTranscript(raw);
|
|
39
|
+
|
|
40
|
+
expect(events).toEqual([{ type: "skill.loaded", skill: "ms-office-suite:pdf", callId: "toolu_02" }]);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
it("工具名大小写不定也能识别(防御性小写比较)", () => {
|
|
44
|
+
const raw = toolUseLine({ id: "toolu_03", name: "skill", input: { skill: "xlsx" } });
|
|
45
|
+
|
|
46
|
+
const { events } = parseClaudeCodeTranscript(raw);
|
|
47
|
+
|
|
48
|
+
expect(events).toEqual([{ type: "skill.loaded", skill: "xlsx", callId: "toolu_03" }]);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it("Skill 加载的 tool_result 被吃掉,不补发孤儿 action.result", () => {
|
|
52
|
+
const raw = [
|
|
53
|
+
toolUseLine({ id: "toolu_01", name: "Skill", input: { skill: "pdf" } }),
|
|
54
|
+
toolResultLine("toolu_01", "skill body here"),
|
|
55
|
+
line({ type: "assistant", message: { content: [{ type: "text", text: "done" }] } }),
|
|
56
|
+
].join("\n");
|
|
57
|
+
|
|
58
|
+
const { events } = parseClaudeCodeTranscript(raw);
|
|
59
|
+
|
|
60
|
+
expect(events.map((e) => e.type)).toEqual(["skill.loaded", "message"]);
|
|
61
|
+
expect(events.some((e) => e.type === "action.result")).toBe(false);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it("input.skill 缺失或非字符串时不当作 Skill 加载,回落成普通 action.called", () => {
|
|
65
|
+
const raw = [
|
|
66
|
+
toolUseLine({ id: "toolu_04", name: "Skill", input: {} }),
|
|
67
|
+
toolUseLine({ id: "toolu_05", name: "Skill", input: { skill: 42 } }),
|
|
68
|
+
].join("\n");
|
|
69
|
+
|
|
70
|
+
const { events } = parseClaudeCodeTranscript(raw);
|
|
71
|
+
|
|
72
|
+
expect(events).toEqual([
|
|
73
|
+
{ type: "action.called", callId: "toolu_04", name: "Skill", input: {}, tool: "unknown" },
|
|
74
|
+
{ type: "action.called", callId: "toolu_05", name: "Skill", input: { skill: 42 }, tool: "unknown" },
|
|
75
|
+
]);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("普通(非 Skill)工具调用完全不受影响:Bash 仍产出 action.called + action.result,顺序不变", () => {
|
|
79
|
+
const raw = [
|
|
80
|
+
toolUseLine({ id: "toolu_06", name: "Bash", input: { command: "ls" } }),
|
|
81
|
+
toolResultLine("toolu_06", "a.ts\nb.ts", false),
|
|
82
|
+
].join("\n");
|
|
83
|
+
|
|
84
|
+
const { events } = parseClaudeCodeTranscript(raw);
|
|
85
|
+
|
|
86
|
+
expect(events).toEqual([
|
|
87
|
+
{ type: "action.called", callId: "toolu_06", name: "Bash", input: { command: "ls" }, tool: "shell" },
|
|
88
|
+
{ type: "action.result", callId: "toolu_06", output: "a.ts\nb.ts", status: "completed" },
|
|
89
|
+
]);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it("Skill 加载与普通工具调用混合出现时,各自独立、顺序保持事件出现顺序", () => {
|
|
93
|
+
const raw = [
|
|
94
|
+
toolUseLine({ id: "t1", name: "Skill", input: { skill: "pdf" } }),
|
|
95
|
+
toolResultLine("t1", "skill body"),
|
|
96
|
+
toolUseLine({ id: "t2", name: "Read", input: { file_path: "/a.txt" } }),
|
|
97
|
+
toolResultLine("t2", "file contents"),
|
|
98
|
+
].join("\n");
|
|
99
|
+
|
|
100
|
+
const { events } = parseClaudeCodeTranscript(raw);
|
|
101
|
+
|
|
102
|
+
expect(events).toEqual([
|
|
103
|
+
{ type: "skill.loaded", skill: "pdf", callId: "t1" },
|
|
104
|
+
{ type: "action.called", callId: "t2", name: "Read", input: { file_path: "/a.txt" }, tool: "file_read" },
|
|
105
|
+
{ type: "action.result", callId: "t2", output: "file contents", status: "completed" },
|
|
106
|
+
]);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it("Read/Write/Edit/WebFetch/WebSearch/Task 等既有别名归一不受 Skill 识别改动影响(回归锁)", () => {
|
|
110
|
+
const raw = [
|
|
111
|
+
toolUseLine({ id: "a", name: "Write", input: { file_path: "/x", content: "y" } }),
|
|
112
|
+
toolUseLine({ id: "b", name: "Task", input: { prompt: "go" } }),
|
|
113
|
+
toolUseLine({ id: "c", name: "WebSearch", input: { query: "q" } }),
|
|
114
|
+
].join("\n");
|
|
115
|
+
|
|
116
|
+
const { events } = parseClaudeCodeTranscript(raw);
|
|
117
|
+
const tools = events.filter((e) => e.type === "action.called").map((e) => (e as { tool?: string }).tool);
|
|
118
|
+
|
|
119
|
+
expect(tools).toEqual(["file_write", "agent_task", "web_search"]);
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
describe("parseClaudeCodeTranscript — 既有行为回归", () => {
|
|
124
|
+
it("assistant 文本 + thinking + tool_use 混合一行时,三者按既有顺序全部产出", () => {
|
|
125
|
+
const raw = line({
|
|
126
|
+
type: "assistant",
|
|
127
|
+
message: {
|
|
128
|
+
content: [
|
|
129
|
+
{ type: "text", text: "checking files" },
|
|
130
|
+
{ type: "thinking", thinking: "should look at package.json" },
|
|
131
|
+
{ type: "tool_use", id: "c1", name: "Bash", input: { command: "cat package.json" } },
|
|
132
|
+
],
|
|
133
|
+
},
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
const { events } = parseClaudeCodeTranscript(raw);
|
|
137
|
+
|
|
138
|
+
expect(events).toEqual([
|
|
139
|
+
{ type: "message", role: "assistant", text: "checking files" },
|
|
140
|
+
{ type: "thinking", text: "should look at package.json" },
|
|
141
|
+
{ type: "action.called", callId: "c1", name: "Bash", input: { command: "cat package.json" }, tool: "shell" },
|
|
142
|
+
]);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it("失败的 tool_result(is_error)映射成 status: failed", () => {
|
|
146
|
+
const raw = [
|
|
147
|
+
toolUseLine({ id: "c1", name: "Bash", input: { command: "false" } }),
|
|
148
|
+
toolResultLine("c1", "command failed", true),
|
|
149
|
+
].join("\n");
|
|
150
|
+
|
|
151
|
+
const { events } = parseClaudeCodeTranscript(raw);
|
|
152
|
+
|
|
153
|
+
expect(events[1]).toMatchObject({ type: "action.result", callId: "c1", status: "failed" });
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
it("compact_boundary 产出 compaction 事件并计数", () => {
|
|
157
|
+
const raw = line({ type: "system", subtype: "compact_boundary" });
|
|
158
|
+
const { events, compactions } = parseClaudeCodeTranscript(raw);
|
|
159
|
+
expect(events).toEqual([{ type: "compaction" }]);
|
|
160
|
+
expect(compactions).toBe(1);
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
it("空/未定义输入返回空事件流,不抛错", () => {
|
|
164
|
+
expect(parseClaudeCodeTranscript(undefined).events).toEqual([]);
|
|
165
|
+
expect(parseClaudeCodeTranscript("").events).toEqual([]);
|
|
166
|
+
expect(parseClaudeCodeTranscript(" \n \n").events).toEqual([]);
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
it("坏 JSON 行不中断解析,标 parseSuccess: false 但保留其余行的事件", () => {
|
|
170
|
+
const raw = ["not json {{{", toolUseLine({ id: "c1", name: "Bash", input: {} })].join("\n");
|
|
171
|
+
const { events, parseSuccess } = parseClaudeCodeTranscript(raw);
|
|
172
|
+
expect(parseSuccess).toBe(false);
|
|
173
|
+
expect(events).toEqual([{ type: "action.called", callId: "c1", name: "Bash", input: {}, tool: "shell" }]);
|
|
174
|
+
});
|
|
175
|
+
});
|
|
@@ -40,6 +40,24 @@ function normalizeToolName(name: string): ToolName {
|
|
|
40
40
|
return normalizeShared(name, CLAUDE_TOOL_ALIASES);
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
+
// ───────────────────────── Skill 加载识别 ─────────────────────────
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Claude Code 原生调用 Skill 时,tool_use 块的 name 恒为 "Skill"(核对过 Claude Code CLI
|
|
47
|
+
* 自带的 SkillTool 定义:`buildTool({ name: SKILL_TOOL_NAME, ... })`,`SKILL_TOOL_NAME = "Skill"`,
|
|
48
|
+
* 严格大小写;这里放宽成大小写不敏感比较,只是防御性兜底,不代表原生真的会变大小写),
|
|
49
|
+
* input 的 schema 是 `z.object({ skill: z.string(), args: z.string().optional() })`——
|
|
50
|
+
* skill 名在 `input.skill`(不是 `command`),可能是限定名如 "ms-office-suite:pdf"。
|
|
51
|
+
*/
|
|
52
|
+
const SKILL_TOOL_NAME = "skill";
|
|
53
|
+
|
|
54
|
+
/** name 是 Skill 工具、且 input.skill 是非空字符串时返回 skill 名;否则 undefined(交给调用方走普通 action.called)。 */
|
|
55
|
+
function extractSkillName(name: string, input: JsonValue): string | undefined {
|
|
56
|
+
if (name.toLowerCase() !== SKILL_TOOL_NAME) return undefined;
|
|
57
|
+
const skill = get(input, "skill");
|
|
58
|
+
return typeof skill === "string" && skill ? skill : undefined;
|
|
59
|
+
}
|
|
60
|
+
|
|
43
61
|
// ───────────────────────── 小工具 ─────────────────────────
|
|
44
62
|
|
|
45
63
|
function get(obj: unknown, key: string): unknown {
|
|
@@ -130,6 +148,10 @@ export function parseClaudeCodeTranscript(raw: string | undefined): ParsedTransc
|
|
|
130
148
|
let requests = 0;
|
|
131
149
|
let compactions = 0;
|
|
132
150
|
let parseSuccess = true;
|
|
151
|
+
// 已识别成 skill.loaded 的 tool_use callId:对应的 tool_result 回来时要吃掉、
|
|
152
|
+
// 不再补发一条 action.result(否则会凭空多出一个没有 action.called 配对的孤儿事件,
|
|
153
|
+
// ExecutionTree 会把它当成占位工具调用节点,而 Skill 加载已经由 skill.loaded 表达过了)。
|
|
154
|
+
const skillCallIds = new Set<string>();
|
|
133
155
|
|
|
134
156
|
if (!raw || !raw.trim()) {
|
|
135
157
|
return { events, usage: { inputTokens: 0, outputTokens: 0 }, compactions: 0, parseSuccess: true };
|
|
@@ -175,6 +197,7 @@ export function parseClaudeCodeTranscript(raw: string | undefined): ParsedTransc
|
|
|
175
197
|
if (toolResults.length > 0) {
|
|
176
198
|
for (const r of toolResults) {
|
|
177
199
|
const callId = String(get(r, "tool_use_id") ?? get(r, "id") ?? "unknown");
|
|
200
|
+
if (skillCallIds.has(callId)) continue; // Skill 加载的结果已经由 skill.loaded 表达过,不重复计入。
|
|
178
201
|
const isError = get(r, "is_error") === true || !!get(r, "error");
|
|
179
202
|
events.push({
|
|
180
203
|
type: "action.result",
|
|
@@ -203,6 +226,14 @@ export function parseClaudeCodeTranscript(raw: string | undefined): ParsedTransc
|
|
|
203
226
|
const name = String(get(b, "name") ?? "unknown");
|
|
204
227
|
const callId = String(get(b, "id") ?? "unknown");
|
|
205
228
|
const input = (get(b, "input") ?? {}) as JsonValue;
|
|
229
|
+
|
|
230
|
+
const skill = extractSkillName(name, input);
|
|
231
|
+
if (skill !== undefined) {
|
|
232
|
+
events.push({ type: "skill.loaded", skill, callId });
|
|
233
|
+
skillCallIds.add(callId);
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
|
|
206
237
|
events.push({
|
|
207
238
|
type: "action.called",
|
|
208
239
|
callId,
|
|
@@ -214,6 +245,7 @@ export function parseClaudeCodeTranscript(raw: string | undefined): ParsedTransc
|
|
|
214
245
|
}
|
|
215
246
|
} else if (type === "tool_result" || type === "tool_response") {
|
|
216
247
|
const callId = String(get(data, "tool_use_id") ?? get(data, "id") ?? "unknown");
|
|
248
|
+
if (skillCallIds.has(callId)) continue; // 同上:Skill 加载的结果不重复计入 action.result。
|
|
217
249
|
const isError = get(data, "is_error") === true || !!get(data, "error");
|
|
218
250
|
events.push({
|
|
219
251
|
type: "action.result",
|
package/src/o11y/types.ts
CHANGED
|
@@ -54,7 +54,7 @@ export interface InputRequest {
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
/**
|
|
57
|
-
* 标准事件流的词汇(对标 docs/
|
|
57
|
+
* 标准事件流的词汇(对标 docs/feature/adapters/architecture/events.md)。adapter 唯一的硬活就是把
|
|
58
58
|
* 各 agent 五花八门的原始 transcript 映射成 StreamEvent[];映射完,整套断言免费。
|
|
59
59
|
*/
|
|
60
60
|
export type StreamEvent =
|
|
@@ -69,6 +69,11 @@ export type StreamEvent =
|
|
|
69
69
|
output?: JsonValue;
|
|
70
70
|
status: "completed" | "failed" | "rejected";
|
|
71
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* 一次 Skill 加载:一等事件,由 adapter 从原生协议里识别出「这是加载 Skill」并直接产出,
|
|
74
|
+
* 不靠 renderer 按工具名/文本猜。`callId` 仅当原生协议把 Skill 加载表达成可关联的工具调用时才有。
|
|
75
|
+
*/
|
|
76
|
+
| { type: "skill.loaded"; skill: string; callId?: string }
|
|
72
77
|
/** 发起一次子 agent 调用(如 Task 工具、远程 sub-agent);`remoteUrl` 仅远程子 agent 有。 */
|
|
73
78
|
| { type: "subagent.called"; callId: string; name: string; remoteUrl?: string }
|
|
74
79
|
/** 一次子 agent 调用的结果,按 `callId` 与对应的 `subagent.called` 对位。 */
|