@netlify/axis 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +24 -916
  2. package/dist/cli.js +2 -4
  3. package/dist/cli.js.map +1 -1
  4. package/dist/config/validator.d.ts.map +1 -1
  5. package/dist/config/validator.js +2 -20
  6. package/dist/config/validator.js.map +1 -1
  7. package/dist/index.d.ts +1 -1
  8. package/dist/index.d.ts.map +1 -1
  9. package/dist/index.js +1 -1
  10. package/dist/index.js.map +1 -1
  11. package/dist/report-ui/index.html +51 -45
  12. package/dist/reports/writer.d.ts +1 -1
  13. package/dist/reports/writer.d.ts.map +1 -1
  14. package/dist/reports/writer.js +2 -1
  15. package/dist/reports/writer.js.map +1 -1
  16. package/dist/runner/runner.d.ts.map +1 -1
  17. package/dist/runner/runner.js +18 -29
  18. package/dist/runner/runner.js.map +1 -1
  19. package/dist/scoring/index.d.ts +1 -7
  20. package/dist/scoring/index.d.ts.map +1 -1
  21. package/dist/scoring/index.js +1 -16
  22. package/dist/scoring/index.js.map +1 -1
  23. package/dist/scoring/sparse-index.d.ts +2 -2
  24. package/dist/scoring/sparse-index.d.ts.map +1 -1
  25. package/dist/scoring/sparse-index.js +8 -9
  26. package/dist/scoring/sparse-index.js.map +1 -1
  27. package/dist/skills/resolver.d.ts +2 -4
  28. package/dist/skills/resolver.d.ts.map +1 -1
  29. package/dist/skills/resolver.js +4 -10
  30. package/dist/skills/resolver.js.map +1 -1
  31. package/dist/transcript/categorize.d.ts +1 -3
  32. package/dist/transcript/categorize.d.ts.map +1 -1
  33. package/dist/transcript/categorize.js +0 -27
  34. package/dist/transcript/categorize.js.map +1 -1
  35. package/dist/types/config.d.ts +4 -17
  36. package/dist/types/config.d.ts.map +1 -1
  37. package/dist/types/report.d.ts +2 -0
  38. package/dist/types/report.d.ts.map +1 -1
  39. package/dist/types/scenario.d.ts +1 -2
  40. package/dist/types/scenario.d.ts.map +1 -1
  41. package/dist/types/scoring.d.ts +0 -4
  42. package/dist/types/scoring.d.ts.map +1 -1
  43. package/package.json +5 -3
  44. package/dist/docs-site/_astro/cli.DDWZtG0-.css +0 -1
  45. package/dist/docs-site/cli/index.html +0 -18
  46. package/dist/docs-site/configuration/index.html +0 -121
  47. package/dist/docs-site/content-assets.mjs +0 -1
  48. package/dist/docs-site/content-modules.mjs +0 -1
  49. package/dist/docs-site/data-store.json +0 -9
  50. package/dist/docs-site/index.html +0 -69
  51. package/dist/docs-site/quickstart/index.html +0 -59
  52. package/dist/docs-site/running/index.html +0 -87
  53. package/dist/docs-site/scoring/index.html +0 -135
  54. package/dist/report-ui/mock-data.json +0 -298
package/dist/docs-site/scoring/index.html DELETED
@@ -1,135 +0,0 @@
1
- <!DOCTYPE html><html lang="en"> <head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Scoring Framework - AXIS Docs</title><meta name="description" content="Documentation for AXIS, the Agent eXperience Index Score synthetic testing framework for AI agents."><link rel="stylesheet" href="/_astro/cli.DDWZtG0-.css"></head> <body> <header class="site-header"> <a href="/" class="site-logo" aria-label="AXIS home"> <span class="site-logo-mark"><span class="logo-ax">AX</span><span class="logo-i">I</span>S</span> </a> <div class="site-header-links"> <a href="https://github.com/netlify/axis">GitHub</a> <a href="https://www.npmjs.com/package/@netlify/axis">npm</a> </div> <button class="mobile-menu-btn" id="menu-btn" aria-label="Toggle navigation">&#9776;</button> </header> <div class="site-shell"> <div class="sidebar-backdrop" id="sidebar-backdrop"></div> <aside class="sidebar" id="sidebar"> <nav class="sidebar-nav"> <div class="sidebar-section-label">Getting Started</div> <a href="/" class="sidebar-link">What is AXIS</a> <a href="/quickstart" class="sidebar-link">Quick Start</a> <div class="sidebar-section-label">How It Works</div> <a href="/scoring" class="sidebar-link active">Scoring Framework</a> <a href="/running" class="sidebar-link">Running Tests</a> <div class="sidebar-section-label">Reference</div> <a href="/cli" class="sidebar-link">CLI</a> <a href="/configuration" class="sidebar-link">Configuration</a> </nav> </aside> <main class="main-content"> <h1>AXIS Scoring Framework</h1> <p class="lead">
2
- AXIS produces a composite 0 to 100 AXIS Result by evaluating four independent dimensions
3
- of agent performance. Each dimension captures a different aspect of how the agent interacted
4
- with your system.
5
- </p> <h2>The Four Dimensions</h2> <div class="dimension-wheel"> <a href="#agent" class="dimension-label dimension-label-agent" data-segment="agent"> <span class="dimension-pct" style="color: var(--color-agent)">20%</span> <span class="dimension-name">Agent</span> <span class="dimension-desc">Planning, reasoning, self-organization</span> </a> <a href="#goal-achievement" class="dimension-label dimension-label-goal" data-segment="goal"> <span class="dimension-pct" style="color: var(--accent)">40%</span> <span class="dimension-name">Goal Achievement</span> <span class="dimension-desc">Evaluated against your scenario rubric</span> </a> <svg class="dimension-chart" viewBox="0 0 200 200" role="img" aria-label="Scoring dimensions: Goal Achievement 40%, Environment 20%, Service 20%, Agent 20%"> <!-- Goal Achievement — 40%, starts at 12 o'clock --> <circle cx="100" cy="100" r="70" fill="none" stroke="var(--accent)" stroke-width="28" stroke-dasharray="173.7 266.1" stroke-dashoffset="111.1" data-segment="goal"></circle> <!-- Environment — 20% --> <circle cx="100" cy="100" r="70" fill="none" stroke="var(--color-env)" stroke-width="28" stroke-dasharray="85.8 354.0" stroke-dashoffset="375.0" data-segment="env"></circle> <!-- Service — 20% --> <circle cx="100" cy="100" r="70" fill="none" stroke="var(--color-svc)" stroke-width="28" stroke-dasharray="85.8 354.0" stroke-dashoffset="287.0" data-segment="svc"></circle> <!-- Agent — 20% --> <circle cx="100" cy="100" r="70" fill="none" stroke="var(--color-agent)" stroke-width="28" stroke-dasharray="85.8 354.0" stroke-dashoffset="199.0" data-segment="agent"></circle> <!-- Center label — default --> <text class="center-default" x="100" y="95" text-anchor="middle" fill="var(--text-muted)" font-family="Inter, -apple-system, system-ui, sans-serif" font-size="12" font-weight="700" letter-spacing="0.5">AXIS</text> <text class="center-default" x="100" y="112" text-anchor="middle" fill="var(--text-muted)" font-family="Inter, -apple-system, system-ui, sans-serif" font-size="11">Result</text> <!-- Center label — hover (hidden by default) --> <text class="center-pct" x="100" y="98" text-anchor="middle" font-family="Inter, -apple-system, system-ui, sans-serif" font-size="28" font-weight="800" opacity="0">40%</text> <text class="center-name" x="100" y="116" text-anchor="middle" fill="var(--text-muted)" font-family="Inter, -apple-system, system-ui, sans-serif" font-size="10" font-weight="500" letter-spacing="0.3" opacity="0">Goal Achievement</text> </svg> <a href="#service" class="dimension-label dimension-label-svc" data-segment="svc"> <span class="dimension-pct" style="color: var(--color-svc)">20%</span> <span class="dimension-name">Service</span> <span class="dimension-desc">APIs, MCP tools, network requests</span> </a> <a href="#environment" class="dimension-label dimension-label-env" data-segment="env"> <span class="dimension-pct" style="color: var(--color-env)">20%</span> <span class="dimension-name">Environment</span> <span class="dimension-desc">Shell, filesystem, git, build tools</span> </a> </div> <h2>Composite AXIS Result</h2> <p>
6
- The final AXIS Result is the weighted average of all four dimension scores.
7
- </p> <pre><code>AXIS Result = (Goal Achievement x 0.4) + (Environment x 0.2) + (Service x 0.2) + (Agent x 0.2)</code></pre> <p>
8
- All dimension scores are 0 to 100. The composite result is rounded to the nearest whole number.
9
- </p> <h2 id="goal-achievement">Goal Achievement</h2> <p>
10
- Goal Achievement is evaluated by a Judge that reads the full agent transcript and compares
11
- the outcome against the rubric checks you defined in the scenario. Each check receives a
12
- score from 0 to 10, which is scaled to 0 to 100 and combined using the check weights.
13
- </p> <p>
14
- This is the only dimension driven entirely by your rubric. The other three dimensions are
15
- calculated from the agent's interaction transcript.
16
- </p> <h2 id="interaction-signals">Interaction Signals</h2> <p>
17
- The Environment, Service, and Agent dimensions are scored by analyzing every tool interaction
18
- in the agent's transcript. Each interaction is evaluated on five signals.
19
- </p> <table> <thead> <tr> <th>Signal</th> <th>Method</th> <th>What It Measures</th> </tr> </thead> <tbody> <tr> <td><strong>Success</strong></td> <td>Judge</td> <td>Did the interaction complete without errors? Were the results usable?</td> </tr> <tr> <td><strong>Speed</strong></td> <td>Heuristic</td> <td>How long did the interaction take relative to expectations for its category?</td> </tr> <tr> <td><strong>Weight</strong></td> <td>Judge</td> <td>Was the tool invocation right-sized? Did the agent request more or less than needed?</td> </tr> <tr> <td><strong>Relevance</strong></td> <td>Judge</td> <td>Was the tool output relevant and useful for completing the task?</td> </tr> <tr> <td><strong>Necessity</strong></td> <td>Judge</td> <td>Were the interactions in this category actually needed, or were they avoidable?</td> </tr> </tbody> </table> <p> <strong>Judge</strong> signals are evaluated by an LLM that reads the full content of each tool
20
- call and its result. <strong>Heuristic</strong> signals are computed deterministically from
21
- measured values like duration, with no LLM involved.
22
- </p> <h3>Signal Weights by Category</h3> <p>
23
- Each category emphasizes different signals based on what matters most for that type of interaction.
24
- </p> <table> <thead> <tr> <th>Signal</th> <th>Environment</th> <th>Service</th> <th>Agent</th> </tr> </thead> <tbody> <tr> <td>Success</td> <td>0.35</td> <td>0.25</td> <td>0.15</td> </tr> <tr> <td>Speed</td> <td>0.15</td> <td>0.15</td> <td>0.15</td> </tr> <tr> <td>Weight</td> <td>0.15</td> <td>0.20</td> <td>0.20</td> </tr> <tr> <td>Relevance</td> <td>0.15</td> <td>0.20</td> <td>0.25</td> </tr> <tr> <td>Necessity</td> <td>0.20</td> <td>0.20</td> <td>0.25</td> </tr> </tbody> </table> <p>
25
- Environment scores weight Success most heavily because failed shell commands or file operations
26
- are the most direct signal of a poor interaction. Speed is weighted equally across all categories
27
- at 0.15 — the category-specific speed thresholds (below) handle the fact that different operation
28
- types have different expected durations. Agent scores weight Necessity, Weight, and Relevance more
29
- heavily because self-organization quality is best measured by whether the agent's own actions were
30
- purposeful and well-scoped.
31
- </p> <h3>Speed Thresholds</h3> <p>
32
- Speed scores are based on how long each interaction took. Thresholds vary by category because
33
- different types of operations have different expected durations.
34
- </p> <table> <thead> <tr> <th>Category</th> <th>Excellent</th> <th>Good</th> <th>Fair</th> <th>Slow</th> <th>Very Slow</th> </tr> </thead> <tbody> <tr> <td>Environment</td> <td>&le;500ms</td> <td>&le;2s</td> <td>&le;5s</td> <td>&le;10s</td> <td>&gt;10s</td> </tr> <tr> <td>Service</td> <td>&le;2s</td> <td>&le;5s</td> <td>&le;10s</td> <td>&le;25s</td> <td>&gt;25s</td> </tr> <tr> <td>Agent</td> <td>&le;2s</td> <td>&le;5s</td> <td>&le;15s</td> <td>&le;30s</td> <td>&gt;30s</td> </tr> </tbody> </table> <h2>Score Calibration</h2> <p>
35
- Raw signal scores are mapped to a 0 to 100 scale using an S-curve rather than linear scaling.
36
- This means:
37
- </p> <ul> <li>Improving from 20 to 50 is relatively easy (fixing obvious problems).</li> <li>Improving from 80 to 95 requires significant quality gains.</li> <li>A score of 50 represents median performance for that category.</li> </ul> <p>
38
- Speed is aggregated using a <strong>severity-weighted average</strong>: slow interactions pull the
39
- score down disproportionately rather than being hidden by many fast interactions. Other signals
40
- (success, weight, relevance) are weighted by context size, so a failed API call that returns a
41
- large error response influences the score more than a trivial file read.
42
- </p> <h2>Transcript Categorization</h2> <p>
43
- Every tool interaction is classified into a category based on the tool name. This classification
44
- determines which dimension the interaction contributes to.
45
- </p> <h3 id="environment">Environment</h3> <p>OS, filesystem, and dev tooling interactions:</p> <ul> <li><strong>Shell:</strong> bash, shell, terminal, exec.</li> <li><strong>File ops:</strong> read, write, edit, glob, grep, cat, head, tail, find, ls, mkdir, rm, cp, mv.</li> <li><strong>Version control:</strong> git.</li> <li><strong>Package managers:</strong> npm, yarn, pip, cargo, go, brew, apt.</li> <li><strong>Build and test:</strong> make, tsc, docker, kubectl, node, python.</li> </ul> <h3 id="agent">Agent</h3> <p>Self-organization and metacognition:</p> <ul> <li><strong>Tool discovery:</strong> toolsearch, listtoolsets, list_tools.</li> <li><strong>Task management:</strong> taskcreate, taskupdate, tasklist, todo_read, todo_write.</li> <li><strong>Planning:</strong> enterplanmode, exitplanmode.</li> <li><strong>User interaction:</strong> askuserquestion, askfollowupquestion.</li> <li><strong>Skill invocation:</strong> skill.</li> </ul> <h3 id="service">Service</h3> <p>
46
- Everything else: external APIs, MCP tools, network calls, and custom services. Any tool not
47
- matching the environment or agent patterns is classified as a service interaction.
48
- </p> <div class="callout callout-info"> <div class="callout-title">Multi-Category Interactions</div> <p>
49
- Some interactions span categories. For example, running <code>curl</code> via bash is both an
50
- environment interaction (shell command) and a service interaction (network call). Environment
51
- tools that target agent-internal paths (like <code>.claude/</code>) are reclassified as agent
52
- interactions.
53
- </p> </div> <h2>Interpreting Scores</h2> <table> <thead> <tr> <th>Range</th> <th>Interpretation</th> </tr> </thead> <tbody> <tr> <td><strong>90 to 100</strong></td> <td>Excellent. Agent completed the task efficiently with minimal waste.</td> </tr> <tr> <td><strong>75 to 89</strong></td> <td>Good. Task completed with minor inefficiencies or missed optimizations.</td> </tr> <tr> <td><strong>50 to 74</strong></td> <td>Fair. Notable issues in execution quality, speed, or unnecessary operations.</td> </tr> <tr> <td><strong>Below 50</strong></td> <td>Poor. Significant failures, errors, or excessive waste in the execution.</td> </tr> </tbody> </table> <p>
54
- When a category score falls below 75, the CLI displays <strong>score insights</strong> that
55
- identify the weakest signal, helping you understand where the agent struggled.
56
- </p> <script>
57
- const wheel = document.querySelector(".dimension-wheel");
58
- if (wheel) {
59
- const segments = wheel.querySelectorAll(".dimension-chart circle[data-segment]");
60
- const labels = wheel.querySelectorAll(".dimension-label");
61
- const defaults = wheel.querySelectorAll(".center-default");
62
- const pctEl = wheel.querySelector(".center-pct");
63
- const nameEl = wheel.querySelector(".center-name");
64
-
65
- const info = {
66
- goal: { pct: "40%", name: "Goal Achievement", color: "var(--accent)" },
67
- env: { pct: "20%", name: "Environment", color: "var(--color-env)" },
68
- svc: { pct: "20%", name: "Service", color: "var(--color-svc)" },
69
- agent: { pct: "20%", name: "Agent", color: "var(--color-agent)" }
70
- };
71
-
72
- function highlight(seg) {
73
- // Fade/grow segments
74
- segments.forEach(s => {
75
- s.style.opacity = s.dataset.segment === seg ? "1" : "0.15";
76
- s.style.strokeWidth = s.dataset.segment === seg ? "34" : "28";
77
- });
78
- // Swap center text
79
- const d = info[seg];
80
- defaults.forEach(t => { t.style.opacity = "0"; });
81
- pctEl.textContent = d.pct;
82
- pctEl.setAttribute("fill", d.color);
83
- pctEl.style.opacity = "1";
84
- nameEl.textContent = d.name;
85
- nameEl.style.opacity = "1";
86
- }
87
-
88
- function reset() {
89
- segments.forEach(s => { s.style.opacity = "1"; s.style.strokeWidth = "28"; });
90
- labels.forEach(l => { l.style.background = ""; l.style.boxShadow = ""; });
91
- defaults.forEach(t => { t.style.opacity = "1"; });
92
- pctEl.style.opacity = "0";
93
- nameEl.style.opacity = "0";
94
- }
95
-
96
- labels.forEach(label => {
97
- label.addEventListener("mouseenter", () => {
98
- highlight(label.dataset.segment);
99
- label.style.background = "var(--light-gray)";
100
- label.style.boxShadow = "var(--shadow)";
101
- });
102
- label.addEventListener("mouseleave", reset);
103
- });
104
-
105
- segments.forEach(circle => {
106
- circle.addEventListener("mouseenter", () => {
107
- highlight(circle.dataset.segment);
108
- const label = wheel.querySelector(`.dimension-label[data-segment="${circle.dataset.segment}"]`);
109
- if (label) {
110
- label.style.background = "var(--light-gray)";
111
- label.style.boxShadow = "var(--shadow)";
112
- }
113
- });
114
- circle.addEventListener("mouseleave", reset);
115
- circle.addEventListener("click", () => {
116
- const label = wheel.querySelector(`.dimension-label[data-segment="${circle.dataset.segment}"]`);
117
- if (label) label.click();
118
- });
119
- });
120
- }
121
- </script> <footer class="site-footer">
122
- AXIS is maintained by Netlify.
123
- </footer> </main> </div> <script>
124
- const btn = document.getElementById("menu-btn");
125
- const sidebar = document.getElementById("sidebar");
126
- const backdrop = document.getElementById("sidebar-backdrop");
127
-
128
- function toggle() {
129
- sidebar.classList.toggle("open");
130
- backdrop.classList.toggle("open");
131
- }
132
-
133
- btn.addEventListener("click", toggle);
134
- backdrop.addEventListener("click", toggle);
135
- </script> </body> </html>
package/dist/report-ui/mock-data.json DELETED
@@ -1,298 +0,0 @@
1
- {
2
- "version": "0.1.0",
3
- "reportId": "2026-04-23-222213",
4
- "timestamp": "2026-04-23T22:22:13.210Z",
5
- "durationMs": 102590,
6
- "summary": {
7
- "total": 1,
8
- "completed": 1,
9
- "failed": 0,
10
- "averageAxisScore": 74
11
- },
12
- "results": [
13
- {
14
- "scenarioKey": "explain-docs",
15
- "scenarioName": "Fetch and summarize documentation",
16
- "agentName": "claude-sdk",
17
- "durationMs": 29570,
18
- "exitCode": 0,
19
- "file": "scenarios/explain-docs/claude-sdk.json",
20
- "tokenUsage": {
21
- "input": 8,
22
- "output": 1146,
23
- "cacheReadInput": 51619
24
- },
25
- "totalCostUsd": 0.05663694999999999,
26
- "prompt": "Fetch the documentation page at https://docs.netlify.com/build/async-workloads/overview/ and write a detailed summary to summary.md. The summary should cover what the product does, setup steps, and key concepts.",
27
- "rubric": [
28
- { "check": "Agent fetched the documentation URL", "weight": 0.25 },
29
- { "check": "Agent wrote summary.md with substantive content", "weight": 0.25 },
30
- { "check": "Summary covers what the product does", "weight": 0.25 },
31
- { "check": "Summary includes setup steps and key concepts", "weight": 0.25 }
32
- ],
33
- "agentConfig": { "adapter": "claude-sdk", "model": "claude-sonnet-4-5-20250929", "maxTurns": 10 },
34
- "score": {
35
- "axisScore": 74,
36
- "goalAchievement": {
37
- "score": 88,
38
- "criteria": [
39
- {
40
- "check": "Agent fetched the documentation URL",
41
- "weight": 0.25,
42
- "score": 10,
43
- "rationale": "Agent used WebFetch on the exact URL provided and the fetched content matches what I independently retrieved from the live page."
44
- },
45
- {
46
- "check": "Agent wrote summary.md with substantive content",
47
- "weight": 0.25,
48
- "score": 10,
49
- "rationale": "summary.md exists at the expected path with 42 lines of well-structured, substantive markdown including headers, a feature table, and use cases."
50
- },
51
- {
52
- "check": "Summary covers what the product does",
53
- "weight": 0.25,
54
- "score": 10,
55
- "rationale": "The summary clearly explains what async workloads are (background fault-tolerant tasks decoupled from client requests), problems solved (retries, scheduling, workflow complexity, infrastructure overhead), and how they relate to Netlify's serverless platform."
56
- },
57
- {
58
- "check": "Summary includes setup steps and key concepts",
59
- "weight": 0.25,
60
- "score": 5,
61
- "rationale": "Key concepts (durable execution, multi-step workflows, scheduling, event-based triggers) are well covered. However, setup steps are entirely absent \u2014 the actual docs describe team-level (Extensions page) and site-level (Project > Build & Deploy) configuration, which the summary omits."
62
- }
63
- ]
64
- },
65
- "environment": {
66
- "score": 67,
67
- "interactionCount": 1,
68
- "auditedCount": 1,
69
- "dimensions": {
70
- "success": 100,
71
- "speed": 100,
72
- "weight": 100,
73
- "relevance": 60,
74
- "necessity": 100
75
- },
76
- "audits": [
77
- {
78
- "id": 6,
79
- "categories": ["environment"],
80
- "success": 1,
81
- "speed": 1,
82
- "weight": 1,
83
- "contextRelevance": 0.6,
84
- "rationale": "The write operation itself succeeded and was right-sized \u2014 the agent wrote what it intended to write (2.9KB). Weight scores high because the tool was used appropriately for the intended operation. contextRelevance is reduced because the output closely mirrors the fetched content structure rather than distilling it, undermining the summarization goal \u2014 though this is a content quality issue, not a tool operation issue."
85
- }
86
- ],
87
- "necessity": {
88
- "category": "environment",
89
- "score": 1,
90
- "unnecessaryIds": [],
91
- "rationale": "Writing summary.md was the core task deliverable. The write was necessary."
92
- }
93
- },
94
- "service": {
95
- "score": 65,
96
- "interactionCount": 1,
97
- "auditedCount": 1,
98
- "dimensions": {
99
- "success": 70,
100
- "speed": 100,
101
- "weight": 60,
102
- "relevance": 70,
103
- "necessity": 100
104
- },
105
- "audits": [
106
- {
107
- "id": 4,
108
- "categories": ["service"],
109
- "success": 0.7,
110
- "speed": 1,
111
- "weight": 0.6,
112
- "contextRelevance": 0.7,
113
- "rationale": "Fetch succeeded and returned structured content, but 3.0KB for a documentation overview page is suspiciously small, suggesting possible JS-gated content was missed. The content returned appears to be a subset of the full page. Weight is penalized because the prompt was verbose but the returned payload was minimal, indicating the fetch likely hit a lightweight shell rather than full rendered docs."
114
- }
115
- ],
116
- "necessity": {
117
- "category": "service",
118
- "score": 1,
119
- "unnecessaryIds": [],
120
- "rationale": "Fetching the documentation page was required to complete the task. The single fetch was necessary and no retries were made."
121
- }
122
- },
123
- "agent": {
124
- "score": 63,
125
- "interactionCount": 5,
126
- "auditedCount": 0,
127
- "dimensions": {
128
- "success": 100,
129
- "speed": 80,
130
- "weight": 80,
131
- "relevance": 80,
132
- "necessity": 80
133
- },
134
- "audits": [
135
- {
136
- "id": 1,
137
- "categories": ["agent"],
138
- "success": 1,
139
- "speed": 0.8,
140
- "weight": 0.8,
141
- "contextRelevance": 0.8,
142
- "rationale": "default"
143
- },
144
- {
145
- "id": 2,
146
- "categories": ["agent"],
147
- "success": 1,
148
- "speed": 0.8,
149
- "weight": 0.8,
150
- "contextRelevance": 0.8,
151
- "rationale": "default"
152
- },
153
- {
154
- "id": 3,
155
- "categories": ["agent"],
156
- "success": 1,
157
- "speed": 0.8,
158
- "weight": 0.8,
159
- "contextRelevance": 0.8,
160
- "rationale": "default"
161
- },
162
- {
163
- "id": 5,
164
- "categories": ["agent"],
165
- "success": 1,
166
- "speed": 0.8,
167
- "weight": 0.8,
168
- "contextRelevance": 0.8,
169
- "rationale": "default"
170
- },
171
- {
172
- "id": 7,
173
- "categories": ["agent"],
174
- "success": 1,
175
- "speed": 0.8,
176
- "weight": 0.8,
177
- "contextRelevance": 0.8,
178
- "rationale": "default"
179
- }
180
- ],
181
- "necessity": {
182
- "category": "agent",
183
- "score": 0.8,
184
- "unnecessaryIds": [],
185
- "rationale": "default"
186
- }
187
- },
188
- "weights": {
189
- "goal_achievement": 0.4,
190
- "environment": 0.2,
191
- "service": 0.2,
192
- "agent": 0.2
193
- },
194
- "sparseIndex": {
195
- "lines": [
196
- "#1 agent user Fetch the documentation page at https://docs.netlify.com/build/async-workload...",
197
- "#2 agent assistant Fetching the Netlify async workloads documentation page.",
198
- "#3 agent tool_use ToolSearch(query: select:WebFetch, max_results: 1) -> ok, 57B, 211ms",
199
- "#4 service tool_use Fetch(url: https://docs.netlify.com/build/async-workloads/overv...) -> ok, 3.0KB, 10.2s",
200
- "#5 agent assistant Now writing the summary to `summary.md`.",
201
- "#6 env tool_use Write(file_path: /private/var/folders/rw/rqr32pr1037grxt33xs5y6...) -> ok, 2.9KB, 11.7s",
202
- "#7 agent assistant `summary.md` has been written. It covers: - **What async workloads are** \u2014 ba..."
203
- ],
204
- "interactions": [
205
- {
206
- "id": 1,
207
- "entryIndices": [0],
208
- "categories": ["agent"],
209
- "sparseLine": "#1 agent user Fetch the documentation page at https://docs.netlify.com/build/async-workload...",
210
- "toolName": null,
211
- "hasError": false,
212
- "durationMs": 2100,
213
- "contextBytes": 236,
214
- "startMs": 0
215
- },
216
- {
217
- "id": 2,
218
- "entryIndices": [1],
219
- "categories": ["agent"],
220
- "sparseLine": "#2 agent assistant Fetching the Netlify async workloads documentation page.",
221
- "toolName": null,
222
- "hasError": false,
223
- "durationMs": 1300,
224
- "contextBytes": 56,
225
- "startMs": 2100
226
- },
227
- {
228
- "id": 3,
229
- "entryIndices": [2, 3],
230
- "categories": ["agent"],
231
- "sparseLine": "#3 agent tool_use ToolSearch(query: select:WebFetch, max_results: 1) -> ok, 57B, 211ms",
232
- "toolName": "ToolSearch",
233
- "hasError": false,
234
- "durationMs": 211,
235
- "contextBytes": 57,
236
- "startMs": 3400
237
- },
238
- {
239
- "id": 4,
240
- "entryIndices": [4, 5],
241
- "categories": ["service"],
242
- "sparseLine": "#4 service tool_use Fetch(url: https://docs.netlify.com/build/async-workloads/overv...) -> ok, 3.0KB, 10.2s",
243
- "toolName": "Fetch",
244
- "hasError": false,
245
- "durationMs": 10195,
246
- "contextBytes": 3089,
247
- "startMs": 3611
248
- },
249
- {
250
- "id": 5,
251
- "entryIndices": [6],
252
- "categories": ["agent"],
253
- "sparseLine": "#5 agent assistant Now writing the summary to `summary.md`.",
254
- "toolName": null,
255
- "hasError": false,
256
- "durationMs": 2194,
257
- "contextBytes": 40,
258
- "startMs": 13806
259
- },
260
- {
261
- "id": 6,
262
- "entryIndices": [7, 8],
263
- "categories": ["environment"],
264
- "sparseLine": "#6 env tool_use Write(file_path: /private/var/folders/rw/rqr32pr1037grxt33xs5y6...) -> ok, 2.9KB, 11.7s",
265
- "toolName": "Write",
266
- "hasError": false,
267
- "durationMs": 11670,
268
- "contextBytes": 2975,
269
- "startMs": 16000
270
- },
271
- {
272
- "id": 7,
273
- "entryIndices": [9],
274
- "categories": ["agent"],
275
- "sparseLine": "#7 agent assistant `summary.md` has been written. It covers: - **What async workloads are** \u2014 ba...",
276
- "toolName": null,
277
- "hasError": false,
278
- "durationMs": 2000,
279
- "contextBytes": 465,
280
- "startMs": 27670
281
- }
282
- ],
283
- "stats": {
284
- "totalInteractions": 7,
285
- "byCategory": {
286
- "environment": 1,
287
- "service": 1,
288
- "agent": 5
289
- },
290
- "totalErrors": 0,
291
- "totalDurationMs": 22076,
292
- "wallClockMs": 29670
293
- }
294
- }
295
- }
296
- }
297
- ]
298
- }