npm - @cursor/july - Versions diffs - 0.1.111 → 0.1.112 - Mend

@cursor/july 0.1.111 → 0.1.112

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (383) hide show

package/dist/docs/reference/evals.html ADDED Viewed

@@ -0,0 +1,116 @@
+<!DOCTYPE html>
+<html lang="en-US" dir="ltr">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <title>Evals | Agent SDK</title>
+    <meta name="description" content="Reference for eval discovery, cases, assertions, judges, configuration, fixtures, reporters, and runner output.">
+    <meta name="generator" content="VitePress v1.6.4">
+    <link rel="preload stylesheet" href="/docs/assets/style.Cfo2vbik.css" as="style">
+    <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
+    <script type="module" src="/docs/assets/app.DxTdhphC.js"></script>
+    <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
+    <link rel="modulepreload" href="/docs/assets/chunks/framework.BNw1pucY.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/theme.B_7J9ZsV.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/katex.DkKDou_j.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/dagre-KV5264BT.BpKJAeRZ.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/cose-bilkent-S5V4N54A.DVeRXIb6.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/c4Diagram-AHTNJAMY.DDJsntUO.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/flowDiagram-DWJPFMVM.CqS_ZQr4.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/erDiagram-SMLLAGMA.vbDotH3l.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/gitGraphDiagram-UUTBAWPF.D04lnbnr.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/ganttDiagram-T4ZO3ILL.DTLdR4pN.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/infoDiagram-42DDH7IO.tAooImWA.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/pieDiagram-DEJITSTG.CLDw6zIs.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/quadrantDiagram-34T5L4WZ.CYaeeY4c.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/xychartDiagram-5P7HB3ND.nAEhF4bO.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/requirementDiagram-MS252O5E.gMYuRpq2.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/sequenceDiagram-FGHM5R23.BTsCjUDN.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/classDiagram-6PBFFD2Q.BgxOlMHw.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/classDiagram-v2-HSJHXN6E.BgxOlMHw.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/stateDiagram-FHFEXIEX.CftT9mLJ.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/stateDiagram-v2-QKLJ7IA2.-43J68xB.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/journeyDiagram-VCZTEJTY.C3tUgyCg.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/timeline-definition-GMOUNBTQ.DbU3WUNw.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/mindmap-definition-QFDTVHPH.Dcp6cxeu.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/kanban-definition-6JOO6SKY.CtD9-QCe.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/sankeyDiagram-XADWPNL6.CZqyHFbc.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/diagram-TYMM5635.qJHRizHR.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/diagram-MMDJMWI5.Dpztst2S.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/blockDiagram-DXYQGD6D.DKmMaTre.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/diagram-5BDNPKRD.BQOtrd1Z.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/architectureDiagram-Q4EWVU46.CJHGP4ki.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/ishikawaDiagram-UXIWVN3A.ClUsVqnJ.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/vennDiagram-DHZGUBPP.ixsq-q2u.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/diagram-G4DWMVQ6.CSDAhjPI.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/wardleyDiagram-NUSXRM2D.C0ewvgbp.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/virtual_mermaid-config.DDnGl6nM.js">
+    <link rel="modulepreload" href="/docs/assets/reference_evals.md.DNJzM_yf.lean.js">
+    <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
+    <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
+    <link rel="alternate" type="text/plain" href="/docs/llms.txt">
+    <link rel="alternate" type="text/markdown" href="/docs/reference/evals.md">
+  </head>
+  <body>
+    <div id="app"><div class="Layout" data-v-76086e96><!--[--><!--]--><!--[--><span tabindex="-1" data-v-97fb5898></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-97fb5898>Skip to content</a><!--]--><!----><header class="VPNav" data-v-76086e96 data-v-781b1279><div class="VPNavBar" data-v-781b1279 data-v-b33015df><div class="wrapper" data-v-b33015df><div class="container" data-v-b33015df><div class="title" data-v-b33015df><div class="VPNavBarTitle has-sidebar" data-v-b33015df data-v-a0c466dc><a class="title" href="/docs/" data-v-a0c466dc><!--[--><!--]--><!----><span data-v-a0c466dc>Agent SDK</span><!--[--><!--[--><!--[--><!--[--><span class="agent-sdk-version" title="@cursor/july 0.1.112" data-v-97e8f22b>0.1.112</span><!--]--><!--]--><!--]--><!--]--></a></div></div><div class="content" data-v-b33015df><div class="content-body" data-v-b33015df><!--[--><!--]--><div class="VPNavBarSearch search" data-v-b33015df><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-b33015df data-v-8906c751><span id="main-nav-aria-label" class="visually-hidden" data-v-8906c751> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/quickstart.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Quickstart</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/guides/mcp-oauth.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Guides</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/guides/github.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Integrations</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/templates/security-reviewer.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Templates</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/reference/project-layout.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-b33015df data-v-959ed110><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-959ed110 data-v-79ad6d7f data-v-0a895de9><span class="check" data-v-0a895de9><span class="icon" data-v-0a895de9><!--[--><span class="vpi-sun sun" data-v-79ad6d7f></span><span class="vpi-moon moon" data-v-79ad6d7f></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-b33015df data-v-a713f501 data-v-1d7aa189><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-1d7aa189><span class="vpi-more-horizontal icon" data-v-1d7aa189></span></button><div class="menu" data-v-1d7aa189><div class="VPMenu" data-v-1d7aa189 data-v-e34b7e14><!----><!--[--><!--[--><!----><div class="group" data-v-a713f501><div class="item appearance" data-v-a713f501><p class="label" data-v-a713f501>Appearance</p><div class="appearance-action" data-v-a713f501><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-a713f501 data-v-79ad6d7f data-v-0a895de9><span class="check" data-v-0a895de9><span class="icon" data-v-0a895de9><!--[--><span class="vpi-sun sun" data-v-79ad6d7f></span><span class="vpi-moon moon" data-v-79ad6d7f></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-b33015df data-v-cb5d0edb><span class="container" data-v-cb5d0edb><span class="top" data-v-cb5d0edb></span><span class="middle" data-v-cb5d0edb></span><span class="bottom" data-v-cb5d0edb></span></span></button></div></div></div></div><div class="divider" data-v-b33015df><div class="divider-line" data-v-b33015df></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-76086e96 data-v-6f794310><div class="container" data-v-6f794310><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-6f794310><span class="vpi-align-left menu-icon" data-v-6f794310></span><span class="menu-text" data-v-6f794310>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-6f794310 data-v-bab84d9c><button data-v-bab84d9c>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-76086e96 data-v-4daa293d><div class="curtain" data-v-4daa293d></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-4daa293d><span class="visually-hidden" id="sidebar-aria-label" data-v-4daa293d> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Getting started</h2><!----></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/quickstart.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Building your first agent</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Integrations</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/github.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>GitHub</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/slack.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Slack</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/webhooks.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Webhooks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/convert-automation.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Cursor Automations</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/opentelemetry.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>OpenTelemetry</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/gitlab.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>GitLab</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/bitbucket.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Bitbucket</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Extensions</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/extensions.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/cloud-agents.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Cloud agents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/grokbot-agents.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Grok Bot agents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/improve.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Self-improvement</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Guides</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/mcp-oauth.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>MCP</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/agent-to-agent.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Agent-to-agent</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/deployment.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Deployment</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/evals.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Evals</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/hillclimbing.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Hillclimbing</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/building-with-agents.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Build with a coding agent</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/troubleshooting.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Troubleshooting</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Templates</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/security-reviewer.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Security reviewer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/thermo-review.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Thermo review</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/thermo-quality-review.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Thermo quality review</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/agentic-owners.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Agentic Approver</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/pr-autofixer.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>PR autofixer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/triage.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Triage agent</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible has-active" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Reference</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/project-layout.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Project layout</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/agent-config.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Agent config</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/instructions.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Instructions</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/tools.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Tools</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/prompt.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>prompt</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/skills.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Skills</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/connections.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>MCP Connections</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/subagents.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Subagents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/channels.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Channels</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/schedules.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Schedules & reminders</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/hooks.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Hooks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/evals.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Evals</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/artifacts.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Artifacts</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/sessions.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Sessions & streaming</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/playground.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Playground</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/cli.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>CLI</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/http-api.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>HTTP API</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-76086e96 data-v-258cbba4><div class="VPDoc has-sidebar has-aside" data-v-258cbba4 data-v-afe580da><!--[--><!--]--><div class="container" data-v-afe580da><div class="aside" data-v-afe580da><div class="aside-curtain" data-v-afe580da></div><div class="aside-container" data-v-afe580da><div class="aside-content" data-v-afe580da><div class="VPDocAside" data-v-afe580da data-v-fb5275a4><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-fb5275a4 data-v-b3783f18><div class="content" data-v-b3783f18><div class="outline-marker" data-v-b3783f18></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-b3783f18>On this page</div><ul class="VPDocOutlineItem root" data-v-b3783f18 data-v-72702c6e><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-fb5275a4></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-afe580da><div class="content-container" data-v-afe580da><!--[--><!--]--><main class="main" data-v-afe580da><div style="position:relative;" class="vp-doc _docs_reference_evals" data-v-afe580da><div><h1 id="evals-reference" tabindex="-1">Evals reference <a class="header-anchor" href="#evals-reference" aria-label="Permalink to &quot;Evals reference&quot;"></a></h1><p>This page is the complete authoring and runner contract for <code>@cursor/july/evals</code>. Start with the <a href="./../evals.html">Evals guide</a> for the workflow and first regression case.</p><h2 id="discovery-and-case-ids" tabindex="-1">Discovery and case IDs <a class="header-anchor" href="#discovery-and-case-ids" aria-label="Permalink to &quot;Discovery and case IDs&quot;"></a></h2><p>Eval files live under the project-root <code>evals/</code> directory and end in <code>.eval.ts</code> or <code>.eval.js</code>. The path under that directory becomes the eval ID:</p><ul><li><code>evals/readiness.eval.ts</code> becomes <code>readiness</code>.</li><li><code>evals/builds/api.eval.ts</code> becomes <code>builds/api</code>.</li><li><code>evals/builds/index.eval.ts</code> becomes <code>builds</code>.</li></ul><p>Export one <code>defineEval</code> with either <code>test</code> or <code>cases</code>, not both. A case ID appends one path segment to the file ID:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEval</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  tags: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;smoke&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">],</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  cases: [</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    {</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      id: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;checkout&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">      async</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">t</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">        await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Review checkout PR 42.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">        t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    {</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      id: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;search&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">      async</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">t</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">        await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Review search PR 7.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">        t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ],</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p>Those cases are <code>prs/checkout</code> and <code>prs/search</code> when the file is <code>evals/prs.eval.ts</code>. Case IDs must be unique single path segments.</p><p>A file can also export an array of <code>defineEval</code> calls. The runner names them with zero-padded indexes such as <code>sql/0000</code>. Use named <code>cases</code> for handwritten scenarios and arrays for loaded datasets.</p><p><code>iterations</code> repeats one datapoint from 1 to 100 times. For <code>iterations: 3</code>, <code>weather/nyc</code> expands to <code>weather/nyc/1</code>, <code>weather/nyc/2</code>, and <code>weather/nyc/3</code>; selecting <code>weather/nyc</code> runs all three.</p><h2 id="configuration" tabindex="-1">Configuration <a class="header-anchor" href="#configuration" aria-label="Permalink to &quot;Configuration&quot;"></a></h2><p>Every running suite needs <code>evals/evals.config.ts</code> with <code>maxConcurrency</code>:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineEvalConfig } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/evals&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEvalConfig</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  maxConcurrency: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">20</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  timeoutMs: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">180_000</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  judge: { model: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;gpt-5.4-mini&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><table tabindex="0"><thead><tr><th>Option</th><th>Contract</th></tr></thead><tbody><tr><td><code>maxConcurrency</code></td><td>Required; concurrent authored datapoints, from 1 to 200</td></tr><tr><td><code>timeoutMs</code></td><td>Per-case timeout; case/file, CLI, then config precedence</td></tr><tr><td><code>judge</code></td><td>Default model for <code>t.judge</code></td></tr><tr><td><code>reporters</code></td><td>Objects notified as cases and runs complete</td></tr><tr><td><code>maxPlaygroundRuns</code></td><td>Number of server-side batches kept in playground history</td></tr></tbody></table><p>A case can override <code>description</code>, <code>tags</code>, <code>timeoutMs</code>, <code>iterations</code>, <code>judge</code>, <code>reporters</code>, and <code>metadata</code>. Case metadata merges over file-level metadata; case reporters add to the file&#39;s reporters.</p><h2 id="drive-turns-with-t-send" tabindex="-1">Drive turns with <code>t.send</code> <a class="header-anchor" href="#drive-turns-with-t-send" aria-label="Permalink to &quot;Drive turns with `t.send`&quot;"></a></h2><p><code>await t.send(message, options?)</code> runs one turn and waits until it finishes, fails, or parks for approval. Several sends in one test share the session.</p><p>The returned turn exposes its assistant <code>message</code>, <code>sessionId</code>, <code>events</code>, ordered <code>toolCalls</code>, <code>ok</code>, and turn <code>index</code>. Assertions on the turn inspect only that turn; assertions on <code>t</code> inspect the whole run. Useful run values include <code>t.reply</code>, <code>t.events</code>, <code>t.turns</code>, <code>t.sessionId</code>, and the timeout <code>t.signal</code>.</p><p>These options apply on the first send because they shape the session:</p><table tabindex="0"><thead><tr><th>Option</th><th>Contract</th></tr></thead><tbody><tr><td><code>workspaceFiles</code></td><td>Relative path-to-content map seeded into the session workspace</td></tr><tr><td><code>workspaceDir</code></td><td>Absolute local harness working directory</td></tr><tr><td><code>cloud</code></td><td>Cloud session options merged over the agent&#39;s static cloud config</td></tr></tbody></table><p>Use <code>turn.expectOk()</code> when later test steps depend on that turn succeeding.</p><h2 id="trajectory-assertions" tabindex="-1">Trajectory assertions <a class="header-anchor" href="#trajectory-assertions" aria-label="Permalink to &quot;Trajectory assertions&quot;"></a></h2><p>Assertions record failures and let the test continue, so one case reports every violated contract.</p><table tabindex="0"><thead><tr><th>Assertion</th><th>Checks</th></tr></thead><tbody><tr><td><code>t.succeeded()</code></td><td>Run finished without failure or an unanswered approval</td></tr><tr><td><code>t.parked()</code></td><td>Run stopped on an unanswered approval</td></tr><tr><td><code>t.calledTool(name, matcher?)</code></td><td>Matching tool request occurred</td></tr><tr><td><code>t.notCalledTool(name)</code></td><td>No request for that tool occurred</td></tr><tr><td><code>t.loadedSkill(name)</code></td><td>Agent opened a named skill</td></tr><tr><td><code>t.toolOrder(names)</code></td><td>Tool requests appeared in relative order</td></tr><tr><td><code>t.usedNoTools()</code></td><td>No tool was requested</td></tr><tr><td><code>t.maxToolCalls(max)</code></td><td>Tool-call count stayed under the bound</td></tr><tr><td><code>t.noFailedActions()</code></td><td>No tool result failed</td></tr><tr><td><code>t.calledSubagent(name, matcher?)</code></td><td>Matching subagent call occurred</td></tr><tr><td><code>t.taggedArtifact(kind?, predicate?)</code></td><td>Matching durable artifact was tagged</td></tr><tr><td><code>t.event(type, matcher?)</code></td><td>Matching session event occurred</td></tr><tr><td><code>t.notEvent(type, matcher?)</code></td><td>Matching session event did not occur</td></tr><tr><td><code>t.eventOrder(matchers)</code></td><td>Event groups appeared in relative order</td></tr><tr><td><code>t.eventsSatisfy(label, predicate)</code></td><td>Predicate accepted the event stream</td></tr><tr><td><code>t.check(value, expectation)</code></td><td>Value met an expectation builder</td></tr><tr><td><code>t.score(name, value)</code></td><td>Recorded a numeric score from 0 to 1</td></tr></tbody></table><p><code>t.requireToolCall</code>, <code>t.requireInputRequest</code>, and <code>t.require</code> return the matched value and stop the remaining test body when no match exists.</p><h2 id="tool-and-event-matchers" tabindex="-1">Tool and event matchers <a class="header-anchor" href="#tool-and-event-matchers" aria-label="Permalink to &quot;Tool and event matchers&quot;"></a></h2><p>Tool matchers can narrow by <code>input</code>, <code>output</code>, <code>status</code>, and <code>count</code>. Values accept literals, regular expressions, or predicates. Object literals partial-deep-match:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;inspect_pr&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, {</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  input: { verdict: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;review&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  status: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;completed&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  count: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p>Tool status is <code>completed</code>, <code>failed</code>, <code>pending</code>, or <code>rejected</code>. Subagent matchers accept <code>output</code>, <code>status</code>, <code>count</code>, and <code>callId</code>. Event matchers accept <code>data</code> and <code>count</code>.</p><h2 id="expectation-builders" tabindex="-1">Expectation builders <a class="header-anchor" href="#expectation-builders" aria-label="Permalink to &quot;Expectation builders&quot;"></a></h2><p>Import builders from <code>@cursor/july/evals</code>:</p><table tabindex="0"><thead><tr><th>Builder</th><th>Default severity</th></tr></thead><tbody><tr><td>`includes(string</td><td>RegExp)`</td></tr><tr><td><code>equals(value)</code></td><td>Hard gate</td></tr><tr><td><code>matches(schema)</code></td><td>Hard gate</td></tr><tr><td><code>satisfies(predicate, label)</code></td><td>Hard gate</td></tr><tr><td><code>similarity(expected)</code></td><td>Soft score</td></tr></tbody></table><p><code>matches</code> accepts Standard Schema implementations such as Zod. <code>normalizedSimilarity(value, expected)</code> returns the same 0-to-1 value used by <code>similarity</code>.</p><h2 id="severity-and-verdicts" tabindex="-1">Severity and verdicts <a class="header-anchor" href="#severity-and-verdicts" aria-label="Permalink to &quot;Severity and verdicts&quot;"></a></h2><p>Every assertion returns a severity handle:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;search&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">soft</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(t.reply, </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">similarity</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Expected answer&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">atLeast</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.8</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.judge.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">closedQA</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;names the failing step&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">gate</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><ul><li><code>.gate(threshold?)</code> fails the case when it misses.</li><li><code>.soft(threshold?)</code> records a value without failing.</li><li><code>.atLeast(threshold)</code> records a soft bar; a miss produces a <code>scored</code> verdict.</li></ul><p>Case verdicts are <code>passed</code>, <code>failed</code>, <code>scored</code>, or <code>skipped</code>. The CLI exits 1 for a failed gate. A scored result exits 0 unless <code>--strict</code> is set. Exit 2 means the selection matched no cases.</p><h2 id="judges" tabindex="-1">Judges <a class="header-anchor" href="#judges" aria-label="Permalink to &quot;Judges&quot;"></a></h2><p>Use a judge for meaning that deterministic checks cannot express:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.judge.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">factuality</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(expected).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">atLeast</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.8</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.judge.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">summarizes</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(expected).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">atLeast</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.8</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.judge.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">closedQA</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;names the root cause&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">gate</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.judge.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">sql</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(expected).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">atLeast</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.9</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><p>Each grades <code>t.reply</code> by default; pass <code>{ on }</code> to grade another value. Judge configuration can live in the project config, eval, case, or call, with the nearest value winning.</p><p><code>t.judge.model(prompt)</code> returns a raw model reply for custom grading. Treat agent output as untrusted input: wrap it with <code>fenceUntrusted</code> and include <code>EVAL_JUDGE_INJECTION_GUARD</code> in the judge prompt.</p><h2 id="datasets-and-fixtures" tabindex="-1">Datasets and fixtures <a class="header-anchor" href="#datasets-and-fixtures" aria-label="Permalink to &quot;Datasets and fixtures&quot;"></a></h2><p>Load committed JSON, JSONL, or YAML relative to the project root:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineEval, includes } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/evals&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { loadYaml } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/evals/loaders&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> rows</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> loadYaml</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">&lt;</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  Array</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">&lt;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{ </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">task</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: string; </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">prompt</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: string; </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">expected</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: string }</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">&gt;</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">&gt;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;evals/data/cases.yaml&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> rows.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">map</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">row</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =&gt;</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">  defineEval</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    description: row.task,</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">    async</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">t</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">      await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(row.prompt);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(t.reply, </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">includes</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(row.expected));</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  })</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><p>Materialize API-backed evidence before running a large suite. Commit the exact payload, diff, or metadata revision and seed it with <code>workspaceFiles</code>.</p><h2 id="reporters-and-results" tabindex="-1">Reporters and results <a class="header-anchor" href="#reporters-and-results" aria-label="Permalink to &quot;Reporters and results&quot;"></a></h2><p>Built-in reporters include <code>JUnit({ filePath, suiteName? })</code> and <code>Artifacts({ dir })</code>. Custom reporters can expose <code>onRunStart</code>, <code>onEvalComplete</code>, and <code>onRunComplete</code>. Reporter errors are logged and do not change the eval verdict.</p><p>JSON output contains run totals plus one result per case. A result can include the case ID, verdict, assertions, session ID, inputs, tool calls, metrics, logs, duration, metadata, tags, final text, and error or skip details.</p><h2 id="cli-selection-and-artifacts" tabindex="-1">CLI selection and artifacts <a class="header-anchor" href="#cli-selection-and-artifacts" aria-label="Permalink to &quot;CLI selection and artifacts&quot;"></a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> builds/checkout</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> smoke</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --verbose</span></span></code></pre></div><p>ID filters match an exact ID and its descendants. Repeated IDs use OR; repeated tags use OR. When both are present, a case must match both groups.</p><p>The local runner starts an ephemeral server. Default artifacts go under <code>evals/&lt;stamp&gt;/</code> in the project state directory and include <code>summary.json</code>, <code>results.jsonl</code>, and <code>evals/&lt;case-id&gt;.json</code>. <code>--artifacts &lt;dir&gt;</code> chooses another destination; <code>--no-artifacts</code> disables them. This location is independent of <code>--state-root</code>.</p><p>Model and judge turns resolve credentials in this order: <code>CURSOR_API_KEY</code>, <code>CURSOR_API_KEY_FILE</code>, <code>CURSOR_SERVICE_ACCOUNT_KEY</code>, then <code>agent-sdk login</code>. Listing cases needs no credential.</p><h2 id="playground-and-hosted-runs" tabindex="-1">Playground and hosted runs <a class="header-anchor" href="#playground-and-hosted-runs" aria-label="Permalink to &quot;Playground and hosted runs&quot;"></a></h2><p>The playground <strong>Evals</strong> view starts batches on its running server and keeps their sessions in the normal session list. <code>--url</code> targets a named running server; <code>--prod --slug &lt;slug&gt;</code> targets a hosted deployment.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> smoke</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> status</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">eval-i</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">d</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">&gt;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> cancel</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">eval-i</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">d</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">&gt;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span></span></code></pre></div><p>Server-side batches use the target server&#39;s discovered evals and store results in its playground history. Local-only reporter and concurrency flags do not apply to <code>--url</code> or <code>--prod</code> runs.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;"></a></h2><ul><li><a href="./../evals.html">Evals guide</a>: author and run a regression workflow</li><li><a href="./cli.html#eval">CLI reference</a>: every runner flag</li><li><a href="./sessions.html">Sessions</a>: event vocabulary used by trajectory checks</li><li><a href="./artifacts.html">Artifacts</a>: durable outputs asserted by <code>taggedArtifact</code></li></ul></div></div></main><footer class="VPDocFooter" data-v-afe580da data-v-feab96e0><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-feab96e0><span class="visually-hidden" id="doc-footer-aria-label" data-v-feab96e0>Pager</span><div class="pager" data-v-feab96e0><a class="VPLink link pager-link prev" href="/docs/reference/hooks.html" data-v-feab96e0><!--[--><span class="desc" data-v-feab96e0>Previous page</span><span class="title" data-v-feab96e0>Hooks</span><!--]--></a></div><div class="pager" data-v-feab96e0><a class="VPLink link pager-link next" href="/docs/reference/artifacts.html" data-v-feab96e0><!--[--><span class="desc" data-v-feab96e0>Next page</span><span class="title" data-v-feab96e0>Artifacts</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
+    <script>window.__VP_HASH_MAP__=JSON.parse("{\"building-with-agents.md\":\"BBCx0AUo\",\"deployment.md\":\"D2jQZuFx\",\"evals.md\":\"BYvfZ-PO\",\"guides_agent-to-agent.md\":\"8oDTfu-E\",\"guides_bitbucket.md\":\"mpevW-VP\",\"guides_cloud-agents.md\":\"Cp1O3u-X\",\"guides_convert-automation.md\":\"CqEyfP6Y\",\"guides_github.md\":\"BwpBp3ed\",\"guides_gitlab.md\":\"DaEC3nMk\",\"guides_grokbot-agents.md\":\"CMhZNdEU\",\"guides_improve.md\":\"Bnp4F99w\",\"guides_mcp-oauth.md\":\"bSFakfCY\",\"guides_opentelemetry.md\":\"BKDxQmmd\",\"guides_slack.md\":\"Bo96y42E\",\"guides_webhooks.md\":\"1A72_VEE\",\"hillclimbing.md\":\"D4E1o5Sa\",\"index.md\":\"C-t81M5J\",\"quickstart.md\":\"DAvVhuuU\",\"reference_agent-config.md\":\"DGPyw7ms\",\"reference_artifacts.md\":\"Bu_4HmsD\",\"reference_channels.md\":\"nFWbzAic\",\"reference_cli.md\":\"DLWDz9ij\",\"reference_connections.md\":\"CmyrlXfY\",\"reference_evals.md\":\"DNJzM_yf\",\"reference_extensions.md\":\"Ceq-qT8d\",\"reference_hooks.md\":\"B7uzNENk\",\"reference_http-api.md\":\"CduHavZ2\",\"reference_instructions.md\":\"CU1My5My\",\"reference_playground.md\":\"Ch2d0Iqi\",\"reference_project-layout.md\":\"BGhgpy9V\",\"reference_prompt.md\":\"Ccp0R53H\",\"reference_schedules.md\":\"B2Nm6FaD\",\"reference_sessions.md\":\"1_6Vyv7x\",\"reference_skills.md\":\"DjQkRefx\",\"reference_subagents.md\":\"BHsSMMyO\",\"reference_tools.md\":\"BYzUTeVA\",\"templates_agentic-owners.md\":\"9M575F5C\",\"templates_pr-autofixer.md\":\"ws0DDXDy\",\"templates_security-reviewer.md\":\"KEFYXzfK\",\"templates_thermo-quality-review.md\":\"VNJ_mohX\",\"templates_thermo-review.md\":\"Hi3zWOkP\",\"templates_triage.md\":\"CVGe_FG6\",\"troubleshooting.md\":\"mnfFG2Em\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"Agent SDK\",\"description\":\"Filesystem-first framework for building and serving Cursor agents.\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"sdkVersion\":\"0.1.112\",\"nav\":[{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Guides\",\"link\":\"/guides/mcp-oauth\"},{\"text\":\"Integrations\",\"link\":\"/guides/github\"},{\"text\":\"Templates\",\"link\":\"/templates/security-reviewer\"},{\"text\":\"Reference\",\"link\":\"/reference/project-layout\"}],\"search\":{\"provider\":\"local\"},\"outline\":{\"level\":[2,3]},\"sidebar\":[{\"text\":\"Getting started\",\"items\":[{\"text\":\"Overview\",\"link\":\"/\"},{\"text\":\"Building your first agent\",\"link\":\"/quickstart\"}]},{\"text\":\"Integrations\",\"collapsed\":false,\"items\":[{\"text\":\"GitHub\",\"link\":\"/guides/github\"},{\"text\":\"Slack\",\"link\":\"/guides/slack\"},{\"text\":\"Webhooks\",\"link\":\"/guides/webhooks\"},{\"text\":\"Cursor Automations\",\"link\":\"/guides/convert-automation\"},{\"text\":\"OpenTelemetry\",\"link\":\"/guides/opentelemetry\"},{\"text\":\"GitLab\",\"link\":\"/guides/gitlab\"},{\"text\":\"Bitbucket\",\"link\":\"/guides/bitbucket\"}]},{\"text\":\"Extensions\",\"collapsed\":false,\"items\":[{\"text\":\"Overview\",\"link\":\"/reference/extensions\"},{\"text\":\"Cloud agents\",\"link\":\"/guides/cloud-agents\"},{\"text\":\"Grok Bot agents\",\"link\":\"/guides/grokbot-agents\"},{\"text\":\"Self-improvement\",\"link\":\"/guides/improve\"}]},{\"text\":\"Guides\",\"collapsed\":false,\"items\":[{\"text\":\"MCP\",\"link\":\"/guides/mcp-oauth\"},{\"text\":\"Agent-to-agent\",\"link\":\"/guides/agent-to-agent\"},{\"text\":\"Deployment\",\"link\":\"/deployment\"},{\"text\":\"Evals\",\"link\":\"/evals\"},{\"text\":\"Hillclimbing\",\"link\":\"/hillclimbing\"},{\"text\":\"Build with a coding agent\",\"link\":\"/building-with-agents\"},{\"text\":\"Troubleshooting\",\"link\":\"/troubleshooting\"}]},{\"text\":\"Templates\",\"collapsed\":false,\"items\":[{\"text\":\"Security reviewer\",\"link\":\"/templates/security-reviewer\"},{\"text\":\"Thermo review\",\"link\":\"/templates/thermo-review\"},{\"text\":\"Thermo quality review\",\"link\":\"/templates/thermo-quality-review\"},{\"text\":\"Agentic Approver\",\"link\":\"/templates/agentic-owners\"},{\"text\":\"PR autofixer\",\"link\":\"/templates/pr-autofixer\"},{\"text\":\"Triage agent\",\"link\":\"/templates/triage\"}]},{\"text\":\"Reference\",\"collapsed\":false,\"items\":[{\"text\":\"Project layout\",\"link\":\"/reference/project-layout\"},{\"text\":\"Agent config\",\"link\":\"/reference/agent-config\"},{\"text\":\"Instructions\",\"link\":\"/reference/instructions\"},{\"text\":\"Tools\",\"link\":\"/reference/tools\"},{\"text\":\"prompt\",\"link\":\"/reference/prompt\"},{\"text\":\"Skills\",\"link\":\"/reference/skills\"},{\"text\":\"MCP Connections\",\"link\":\"/reference/connections\"},{\"text\":\"Subagents\",\"link\":\"/reference/subagents\"},{\"text\":\"Channels\",\"link\":\"/reference/channels\"},{\"text\":\"Schedules & reminders\",\"link\":\"/reference/schedules\"},{\"text\":\"Hooks\",\"link\":\"/reference/hooks\"},{\"text\":\"Evals\",\"link\":\"/reference/evals\"},{\"text\":\"Artifacts\",\"link\":\"/reference/artifacts\"},{\"text\":\"Sessions & streaming\",\"link\":\"/reference/sessions\"},{\"text\":\"Playground\",\"link\":\"/reference/playground\"},{\"text\":\"CLI\",\"link\":\"/reference/cli\"},{\"text\":\"HTTP API\",\"link\":\"/reference/http-api\"}]}]},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
+  </body>
+</html>

package/dist/docs/reference/evals.md ADDED Viewed

@@ -0,0 +1,293 @@
+# Evals reference
+This page is the complete authoring and runner contract for
+`@cursor/july/evals`. Start with the [Evals guide](/docs/evals.md) for the
+workflow and first regression case.
+## Discovery and case IDs
+Eval files live under the project-root `evals/` directory and end in
+`.eval.ts` or `.eval.js`. The path under that directory becomes the
+eval ID:
+- `evals/readiness.eval.ts` becomes `readiness`.
+- `evals/builds/api.eval.ts` becomes `builds/api`.
+- `evals/builds/index.eval.ts` becomes `builds`.
+Export one `defineEval` with either `test` or `cases`, not both. A case
+ID appends one path segment to the file ID:
+```ts
+export default defineEval({
+  tags: ["smoke"],
+  cases: [
+    {
+      id: "checkout",
+      async test(t) {
+        await t.send("Review checkout PR 42.");
+        t.succeeded();
+      },
+    },
+    {
+      id: "search",
+      async test(t) {
+        await t.send("Review search PR 7.");
+        t.succeeded();
+      },
+    },
+  ],
+});
+```
+Those cases are `prs/checkout` and `prs/search` when the file is
+`evals/prs.eval.ts`. Case IDs must be unique single path segments.
+A file can also export an array of `defineEval` calls. The runner names
+them with zero-padded indexes such as `sql/0000`. Use named `cases` for
+handwritten scenarios and arrays for loaded datasets.
+`iterations` repeats one datapoint from 1 to 100 times. For
+`iterations: 3`, `weather/nyc` expands to `weather/nyc/1`,
+`weather/nyc/2`, and `weather/nyc/3`; selecting `weather/nyc` runs all
+three.
+## Configuration
+Every running suite needs `evals/evals.config.ts` with
+`maxConcurrency`:
+```ts
+import { defineEvalConfig } from "@cursor/july/evals";
+export default defineEvalConfig({
+  maxConcurrency: 20,
+  timeoutMs: 180_000,
+  judge: { model: "gpt-5.4-mini" },
+});
+```
+| Option | Contract |
+| --- | --- |
+| `maxConcurrency` | Required; concurrent authored datapoints, from 1 to 200 |
+| `timeoutMs` | Per-case timeout; case/file, CLI, then config precedence |
+| `judge` | Default model for `t.judge` |
+| `reporters` | Objects notified as cases and runs complete |
+| `maxPlaygroundRuns` | Number of server-side batches kept in playground history |
+A case can override `description`, `tags`, `timeoutMs`, `iterations`,
+`judge`, `reporters`, and `metadata`. Case metadata merges over
+file-level metadata; case reporters add to the file's reporters.
+## Drive turns with `t.send`
+`await t.send(message, options?)` runs one turn and waits until it
+finishes, fails, or parks for approval. Several sends in one test share
+the session.
+The returned turn exposes its assistant `message`, `sessionId`,
+`events`, ordered `toolCalls`, `ok`, and turn `index`. Assertions on the
+turn inspect only that turn; assertions on `t` inspect the whole run.
+Useful run values include `t.reply`, `t.events`, `t.turns`,
+`t.sessionId`, and the timeout `t.signal`.
+These options apply on the first send because they shape the session:
+| Option | Contract |
+| --- | --- |
+| `workspaceFiles` | Relative path-to-content map seeded into the session workspace |
+| `workspaceDir` | Absolute local harness working directory |
+| `cloud` | Cloud session options merged over the agent's static cloud config |
+Use `turn.expectOk()` when later test steps depend on that turn
+succeeding.
+## Trajectory assertions
+Assertions record failures and let the test continue, so one case
+reports every violated contract.
+| Assertion | Checks |
+| --- | --- |
+| `t.succeeded()` | Run finished without failure or an unanswered approval |
+| `t.parked()` | Run stopped on an unanswered approval |
+| `t.calledTool(name, matcher?)` | Matching tool request occurred |
+| `t.notCalledTool(name)` | No request for that tool occurred |
+| `t.loadedSkill(name)` | Agent opened a named skill |
+| `t.toolOrder(names)` | Tool requests appeared in relative order |
+| `t.usedNoTools()` | No tool was requested |
+| `t.maxToolCalls(max)` | Tool-call count stayed under the bound |
+| `t.noFailedActions()` | No tool result failed |
+| `t.calledSubagent(name, matcher?)` | Matching subagent call occurred |
+| `t.taggedArtifact(kind?, predicate?)` | Matching durable artifact was tagged |
+| `t.event(type, matcher?)` | Matching session event occurred |
+| `t.notEvent(type, matcher?)` | Matching session event did not occur |
+| `t.eventOrder(matchers)` | Event groups appeared in relative order |
+| `t.eventsSatisfy(label, predicate)` | Predicate accepted the event stream |
+| `t.check(value, expectation)` | Value met an expectation builder |
+| `t.score(name, value)` | Recorded a numeric score from 0 to 1 |
+`t.requireToolCall`, `t.requireInputRequest`, and `t.require` return the
+matched value and stop the remaining test body when no match exists.
+## Tool and event matchers
+Tool matchers can narrow by `input`, `output`, `status`, and `count`.
+Values accept literals, regular expressions, or predicates. Object
+literals partial-deep-match:
+```ts
+t.calledTool("inspect_pr", {
+  input: { verdict: "review" },
+  status: "completed",
+  count: 1,
+});
+```
+Tool status is `completed`, `failed`, `pending`, or `rejected`.
+Subagent matchers accept `output`, `status`, `count`, and `callId`.
+Event matchers accept `data` and `count`.
+## Expectation builders
+Import builders from `@cursor/july/evals`:
+| Builder | Default severity |
+| --- | --- |
+| `includes(string | RegExp)` | Hard gate |
+| `equals(value)` | Hard gate |
+| `matches(schema)` | Hard gate |
+| `satisfies(predicate, label)` | Hard gate |
+| `similarity(expected)` | Soft score |
+`matches` accepts Standard Schema implementations such as Zod.
+`normalizedSimilarity(value, expected)` returns the same 0-to-1 value
+used by `similarity`.
+## Severity and verdicts
+Every assertion returns a severity handle:
+```ts
+t.succeeded();
+t.calledTool("search").soft();
+t.check(t.reply, similarity("Expected answer")).atLeast(0.8);
+t.judge.closedQA("names the failing step").gate(1);
+```
+- `.gate(threshold?)` fails the case when it misses.
+- `.soft(threshold?)` records a value without failing.
+- `.atLeast(threshold)` records a soft bar; a miss produces a `scored`
+  verdict.
+Case verdicts are `passed`, `failed`, `scored`, or `skipped`. The CLI
+exits 1 for a failed gate. A scored result exits 0 unless `--strict` is
+set. Exit 2 means the selection matched no cases.
+## Judges
+Use a judge for meaning that deterministic checks cannot express:
+```ts
+t.judge.factuality(expected).atLeast(0.8);
+t.judge.summarizes(expected).atLeast(0.8);
+t.judge.closedQA("names the root cause").gate(1);
+t.judge.sql(expected).atLeast(0.9);
+```
+Each grades `t.reply` by default; pass `{ on }` to grade another value.
+Judge configuration can live in the project config, eval, case, or
+call, with the nearest value winning.
+`t.judge.model(prompt)` returns a raw model reply for custom grading.
+Treat agent output as untrusted input: wrap it with `fenceUntrusted` and
+include `EVAL_JUDGE_INJECTION_GUARD` in the judge prompt.
+## Datasets and fixtures
+Load committed JSON, JSONL, or YAML relative to the project root:
+```ts
+import { defineEval, includes } from "@cursor/july/evals";
+import { loadYaml } from "@cursor/july/evals/loaders";
+const rows = await loadYaml<
+  Array<{ task: string; prompt: string; expected: string }>
+>("evals/data/cases.yaml");
+export default rows.map(row =>
+  defineEval({
+    description: row.task,
+    async test(t) {
+      await t.send(row.prompt);
+      t.succeeded();
+      t.check(t.reply, includes(row.expected));
+    },
+  })
+);
+```
+Materialize API-backed evidence before running a large suite. Commit
+the exact payload, diff, or metadata revision and seed it with
+`workspaceFiles`.
+## Reporters and results
+Built-in reporters include `JUnit({ filePath, suiteName? })` and
+`Artifacts({ dir })`. Custom reporters can expose `onRunStart`,
+`onEvalComplete`, and `onRunComplete`. Reporter errors are logged and
+do not change the eval verdict.
+JSON output contains run totals plus one result per case. A result can
+include the case ID, verdict, assertions, session ID, inputs, tool
+calls, metrics, logs, duration, metadata, tags, final text, and error or
+skip details.
+## CLI selection and artifacts
+```bash
+agent-sdk eval --dir . --list
+agent-sdk eval --dir .
+agent-sdk eval --dir . builds/checkout
+agent-sdk eval --dir . --tag smoke
+agent-sdk eval --dir . --verbose
+```
+ID filters match an exact ID and its descendants. Repeated IDs use OR;
+repeated tags use OR. When both are present, a case must match both
+groups.
+The local runner starts an ephemeral server. Default artifacts go under
+`evals/<stamp>/` in the project state directory and include
+`summary.json`, `results.jsonl`, and `evals/<case-id>.json`.
+`--artifacts <dir>` chooses another destination; `--no-artifacts`
+disables them. This location is independent of `--state-root`.
+Model and judge turns resolve credentials in this order:
+`CURSOR_API_KEY`, `CURSOR_API_KEY_FILE`,
+`CURSOR_SERVICE_ACCOUNT_KEY`, then `agent-sdk login`. Listing cases
+needs no credential.
+## Playground and hosted runs
+The playground **Evals** view starts batches on its running server and
+keeps their sessions in the normal session list. `--url` targets a
+named running server; `--prod --slug <slug>` targets a hosted
+deployment.
+```bash
+agent-sdk eval --prod --slug vulnerability-scanner --tag smoke
+agent-sdk eval status <eval-id> --prod --slug vulnerability-scanner
+agent-sdk eval cancel <eval-id> --prod --slug vulnerability-scanner
+```
+Server-side batches use the target server's discovered evals and store
+results in its playground history. Local-only reporter and concurrency
+flags do not apply to `--url` or `--prod` runs.
+## Related
+- [Evals guide](/docs/evals.md): author and run a regression workflow
+- [CLI reference](/docs/reference/cli.md#eval): every runner flag
+- [Sessions](/docs/reference/sessions.md): event vocabulary used by trajectory checks
+- [Artifacts](/docs/reference/artifacts.md): durable outputs asserted by
+  `taggedArtifact`