@cursor/july 0.1.108 → 0.1.111

package/dist/docs/assets/evals.md.BLDRt5LH.js

@@ -0,0 +1,211 @@
+import{_ as i,c as a,o as e,a3 as t}from"./chunks/framework.dypDpWZ3.js";const c=JSON.parse(`{"title":"Evals","description":"Write defineEval cases that gate an agent's trajectory, run them with agent-sdk eval locally and in CI, and lock every kept improvement with a regression check.","frontmatter":{"title":"Evals","description":"Write defineEval cases that gate an agent's trajectory, run them with agent-sdk eval locally and in CI, and lock every kept improvement with a regression check."},"headers":[],"relativePath":"evals.md","filePath":"evals.md"}`),n={name:"evals.md"};function h(l,s,p,d,k,r){return e(),a("div",null,[...s[0]||(s[0]=[t(`<h1 id="evals" tabindex="-1">Evals <a class="header-anchor" href="#evals" aria-label="Permalink to &quot;Evals&quot;">​</a></h1><p>An eval sends a fixed message to your agent and asserts over the trajectory it records: the turn completed, the right tool ran with the right input, the reply has the right shape. Evals are how you know a prompt tweak helped, a refactor didn&#39;t regress the agent, and last month&#39;s fix still holds.</p><p>Nothing is mocked. The runner starts (or targets) a real agent server, drives sessions over the public API, and grades the events it gets back. The model runs and server tools execute, so <a href="#keep-side-effects-out-of-eval-sessions">keep side effects out of eval sessions</a> before you point an eval at an agent that posts anywhere.</p><h2 id="evals-hooks-or-hillclimbing" tabindex="-1">Evals, hooks, or hillclimbing? <a class="header-anchor" href="#evals-hooks-or-hillclimbing" aria-label="Permalink to &quot;Evals, hooks, or hillclimbing?&quot;">​</a></h2><p>All three read the same session event stream. Pick by the question you are asking.</p><table tabindex="0"><thead><tr><th>You want to</th><th>Use</th></tr></thead><tbody><tr><td>Gate one fixed input&#39;s behavior, locally and in CI</td><td>Evals (this page)</td></tr><tr><td>Observe every live session: metrics, audit, alerts</td><td><a href="./reference/hooks.html">Hooks</a></td></tr><tr><td>Improve an agent one measured round at a time</td><td><a href="./hillclimbing.html">Hillclimbing</a>; each kept win lands an eval</td></tr></tbody></table><p><a href="./reference/hooks.html#hooks-channel-events-or-evals">Hooks, channel events, or evals?</a> has the side-by-side table.</p><h3 id="when-not-to-write-an-eval" tabindex="-1">When not to write an eval <a class="header-anchor" href="#when-not-to-write-an-eval" aria-label="Permalink to &quot;When not to write an eval&quot;">​</a></h3><ul><li>Test a server tool&#39;s own logic with <code>agent-sdk call &lt;tool&gt; --dir . --input &#39;{...}&#39;</code> or a unit test. No model turn, no credential.</li><li>Explore a prompt with <code>agent-sdk run --dir . --message &quot;...&quot;</code> and read the trajectory. Write the eval once you know which decision to gate.</li><li>Stop a bad turn while it runs with <a href="./reference/tools.html#gate-a-tool-on-human-approval"><code>needsApproval</code></a> on the tool or <code>defineResult</code>. Evals grade after the fact.</li></ul><h2 id="write-your-first-eval" tabindex="-1">Write your first eval <a class="header-anchor" href="#write-your-first-eval" aria-label="Permalink to &quot;Write your first eval&quot;">​</a></h2><p>Evals live under the project-root <code>evals/</code> directory, a sibling of <code>agent/</code>. <code>agent/evals/</code> is silently ignored. Discovery loads every <code>.eval.ts</code> (or <code>.eval.js</code>) file under <code>evals/</code>, plus one config file.</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>my-agent/</span></span>
+<span class="line"><span>  agent/</span></span>
+<span class="line"><span>    agent.ts</span></span>
+<span class="line"><span>    tools/inspect_pr.ts</span></span>
+<span class="line"><span>  evals/</span></span>
+<span class="line"><span>    evals.config.ts      # required to run: maxConcurrency</span></span>
+<span class="line"><span>    readiness.eval.ts    # id: readiness</span></span>
+<span class="line"><span>    prs.eval.ts          # cases: prs/checkout, prs/search</span></span></code></pre></div><p>An eval is a single <code>async test(t)</code>. You drive the agent with <code>t.send</code> and assert on the recorded run with the same <code>t</code>:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/readiness.eval.ts</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineEval, includes } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/evals&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEval</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  description: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Inspects a PR without approving it.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  tags: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;smoke&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">],</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  timeoutMs: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">120_000</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">  async</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">t</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">    await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
+<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">      &quot;Is https://github.com/acme/checkout/pull/42 ready to approve?&quot;</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    );</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;inspect_pr&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">notCalledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;approve_pr&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(t.reply, </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">includes</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">ready</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">approve</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">i</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">));</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/evals.config.ts</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineEvalConfig } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/evals&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEvalConfig</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({ maxConcurrency: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">20</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> });</span></span></code></pre></div><p>Run it under Node 22.13 or newer (never Bun) with a Cursor credential in place; see <a href="#credentials">Credentials</a>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> readiness</span></span></code></pre></div><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>PASS  readiness (14.2s) — Inspects a PR without approving it.</span></span>
+<span class="line"><span>  ✓ succeeded</span></span>
+<span class="line"><span>  ✓ calledTool(inspect_pr)</span></span>
+<span class="line"><span>  ✓ notCalledTool(approve_pr)</span></span>
+<span class="line"><span>  ✓ check(includes)</span></span>
+<span class="line"><span></span></span>
+<span class="line"><span>1 passed, 0 failed, 1 total</span></span>
+<span class="line"><span>artifacts: &lt;project state directory&gt;/evals/2026-09-11T15-02-11-402Z</span></span></code></pre></div><p>Every local run writes each case&#39;s assertions, inputs, tool calls, and <code>t.log</code> lines under that artifacts directory. Open <code>evals/&lt;case-id&gt;.json</code> there when a case fails; see <a href="#where-results-land">Where results land</a>.</p><h2 id="name-cases-by-path" tabindex="-1">Name cases by path <a class="header-anchor" href="#name-cases-by-path" aria-label="Permalink to &quot;Name cases by path&quot;">​</a></h2><p>The file path is the eval&#39;s identity, so you don&#39;t author an id. <code>evals/builds/api.eval.ts</code> becomes <code>builds/api</code>. An <code>index</code> filename collapses to its directory: <code>evals/builds/index.eval.ts</code> becomes <code>builds</code>.</p><p>One file can hold several datapoints through <code>cases</code>. Provide either <code>test</code> or <code>cases</code>, not both. Each case id becomes <code>&lt;fileId&gt;/&lt;case.id&gt;</code>:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/prs.eval.ts: prs/checkout, prs/search</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEval</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  tags: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;smoke&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;prs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">],</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  cases: [</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    {</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      id: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;checkout&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      description: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Checkout PR readiness.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">      async</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">t</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">        await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
+<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">          &quot;Is https://github.com/acme/checkout/pull/42 ready to approve?&quot;</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">        );</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">        t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">        t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;inspect_pr&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    {</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      id: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;search&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">      async</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">t</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">        await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
+<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">          &quot;Check https://github.com/acme/search/pull/7 before approval.&quot;</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">        );</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">        t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">        t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;inspect_pr&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ],</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p>Case ids are single path segments, unique within the file. A case can set its own <code>description</code>, <code>tags</code>, <code>timeoutMs</code>, <code>iterations</code>, <code>judge</code>, <code>reporters</code>, and <code>metadata</code>. A case-level value replaces the file-level one for that datapoint, except <code>metadata</code>, which merges with case keys winning, and <code>reporters</code>, which adds to the file&#39;s list. <code>metadata</code> is free-form data carried onto the result and every reporter.</p><p>A file may instead export an array of <code>defineEval</code> calls to fan out over a dataset. Ids are then the file id plus a zero-padded index (<code>sql/0000</code>, <code>sql/0001</code>, ...); see <a href="#load-a-dataset">Load a dataset</a>. Prefer <code>cases</code> when datapoints are hand-written and deserve stable names.</p><h3 id="iterations" tabindex="-1">Iterations <a class="header-anchor" href="#iterations" aria-label="Permalink to &quot;Iterations&quot;">​</a></h3><p><code>iterations</code> (file or case, default <code>1</code>, cap <code>100</code>) runs a datapoint repeatedly. Discovery expands <code>iterations: 3</code> on case <code>nyc</code> to runnable ids <code>weather/nyc/1</code>, <code>weather/nyc/2</code>, and <code>weather/nyc/3</code>. The filter <code>weather/nyc</code> still selects all three. Each expanded case exposes <code>t.iteration</code> and <code>t.iterations</code>.</p><p><code>maxConcurrency</code> counts authored datapoints, not expanded iterations. Iterations of one datapoint share a concurrency slot and run in sequence, so a suite of 11 cases with 3 iterations each and <code>maxConcurrency: 20</code> has at most 11 cases in flight.</p><h2 id="drive-the-agent-with-t-send" tabindex="-1">Drive the agent with <code>t.send</code> <a class="header-anchor" href="#drive-the-agent-with-t-send" aria-label="Permalink to &quot;Drive the agent with \`t.send\`&quot;">​</a></h2><p><code>t.send(message, options?)</code> runs one turn and waits for it to settle: complete, park on an approval request, or fail. Several sends in one case share the session, which is how you write multi-turn evals.</p><p>Each send resolves to a turn result: <code>message</code> (the assistant text), <code>sessionId</code>, <code>events</code>, <code>toolCalls</code> (tool names in order), <code>ok</code>, and <code>index</code>. The turn carries the same assertion vocabulary as <code>t</code>, scoped to that turn, so you can grade an intermediate turn before the next send overwrites <code>t.reply</code>. <code>turn.expectOk()</code> throws when the turn failed, for later steps that depend on it.</p><p>Read the whole case with <code>t.reply</code> (last assistant text), <code>t.events</code> (every event so far), <code>t.turns</code> (settled turns, oldest first), and <code>t.sessionId</code>. <code>t.signal</code> aborts when the case hits its timeout; pass it to your own async work.</p><p>Three options apply on the first send only, because they shape session creation:</p><table tabindex="0"><thead><tr><th>Option</th><th>Effect</th></tr></thead><tbody><tr><td><code>workspaceFiles</code></td><td><code>{ path: contents }</code> seeded into the local session workspace. Prefer this over machine-local paths</td></tr><tr><td><code>workspaceDir</code></td><td>Absolute harness cwd for the local runtime</td></tr><tr><td><code>cloud</code></td><td>Per-session cloud options merged over the agent&#39;s static <code>cloud</code> config. Pin a fixture repo here for cloud evals instead of on the agent&#39;s default <code>cloud.repos</code>. On the cloud runtime, seeded files reach the agent as described under <a href="./reference/agent-config.html#choose-a-runtime">Choose a runtime</a></td></tr></tbody></table><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Review pr/diff.patch and post findings.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, {</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  workspaceFiles: {</span></span>
+<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">    &quot;pr/diff.patch&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
+<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">      &quot;diff --git a/app/routes/search.ts b/app/routes/search.ts&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">      &quot;+res.send(\`&lt;h1&gt;Results for \${req.query.q}&lt;/h1&gt;\`);&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    ].</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">join</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">),</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><h2 id="assert-over-the-trajectory" tabindex="-1">Assert over the trajectory <a class="header-anchor" href="#assert-over-the-trajectory" aria-label="Permalink to &quot;Assert over the trajectory&quot;">​</a></h2><p>Assertions record; they never throw. One run reports every failure instead of dying on the first. Assertions on <code>t</code> read the whole run. Assertions on a turn read only that turn.</p><table tabindex="0"><thead><tr><th>Gate</th><th>Checks</th></tr></thead><tbody><tr><td><code>t.succeeded()</code></td><td>the run did not fail and is not parked on an unanswered approval</td></tr><tr><td><code>t.parked()</code></td><td>the run cleanly parked on an unanswered approval request</td></tr><tr><td><code>t.messageIncludes(token)</code></td><td>the joined assistant text matches a string or <code>RegExp</code></td></tr><tr><td><code>t.calledTool(name, matcher?)</code></td><td>a matching call to <code>name</code> happened</td></tr><tr><td><code>t.notCalledTool(name)</code></td><td>no request for <code>name</code>, in any lifecycle state</td></tr><tr><td><code>t.loadedSkill(name)</code></td><td>the agent opened <code>skills/&lt;name&gt;/SKILL.md</code> (read, grep, or shell <code>cat</code>)</td></tr><tr><td><code>t.toolOrder(names)</code></td><td>tool requests appear in this relative order; extra calls allowed</td></tr><tr><td><code>t.usedNoTools()</code></td><td>no tool calls at all</td></tr><tr><td><code>t.maxToolCalls(max)</code></td><td>at most <code>max</code> tool calls</td></tr><tr><td><code>t.noFailedActions()</code></td><td>no tool call reported an error</td></tr><tr><td><code>t.calledSubagent(name, matcher?)</code></td><td>a matching subagent delegation happened</td></tr><tr><td><code>t.taggedArtifact(kind?, predicate?)</code></td><td>at least one <a href="./reference/artifacts.html">artifact</a> was tagged</td></tr><tr><td><code>t.event(type, matcher?)</code></td><td>at least one matching <a href="./reference/sessions.html#which-events-can-i-stream">event</a> of <code>type</code></td></tr><tr><td><code>t.notEvent(type, matcher?)</code></td><td>no matching event of <code>type</code></td></tr><tr><td><code>t.eventOrder(matchers)</code></td><td>matching event groups occur in this relative order</td></tr><tr><td><code>t.eventsSatisfy(label, predicate)</code></td><td>your predicate over the typed event stream</td></tr><tr><td><code>t.check(value, expectation)</code></td><td>any value, against a <a href="#grade-values-with-expectation-builders">builder</a></td></tr><tr><td><code>t.score(name, value)</code></td><td>a 0-1 score you computed; soft until you add a bar</td></tr></tbody></table><p>Three more assertions gate and return the matched fact. They stop the test body when nothing matches, without a duplicate execution error. <code>t.requireToolCall(name, matcher?)</code> returns the call so later code can read its <code>input</code> and <code>output</code>. <code>t.requireInputRequest(filter?)</code> returns the single pending approval request. <code>await t.require(value, expectation)</code> does the same for a value check.</p><p>A case with no assertions passes when at least one turn completed. Add <code>t.succeeded()</code> and behavior gates anyway. They make the contract visible in review.</p><h3 id="what-good-cases-assert" tabindex="-1">What good cases assert <a class="header-anchor" href="#what-good-cases-assert" aria-label="Permalink to &quot;What good cases assert&quot;">​</a></h3><p>Gate decisions and shape, not prose. Model wording varies run to run. Tool choice, tool avoidance, and output structure are the stable contract.</p><ol><li><code>t.succeeded()</code>: always, first.</li><li>The tool decision: <code>calledTool</code> for the intended path, <code>notCalledTool</code> for the likely wrong alternative. The pair is stronger than either alone.</li><li>Output shape: a regex for the contract (<code>/ready|blocked/i</code>, a JSON marker, a findings-block fence), never exact sentences.</li><li>For structured output, parse <code>t.reply</code> and check fields with <code>matches</code> or <code>satisfies</code> instead of substring-matching JSON.</li></ol><p>The common failure modes: asserting exact phrasing, packing more than about five gates into one case (split it), and cases that depend on live external state that drifts (pin the input).</p><h3 id="narrow-tool-assertions-with-matchers" tabindex="-1">Narrow tool assertions with matchers <a class="header-anchor" href="#narrow-tool-assertions-with-matchers" aria-label="Permalink to &quot;Narrow tool assertions with matchers&quot;">​</a></h3><p>With no matcher, <code>calledTool</code> is request-based: a requested call counts even before its result arrives. A matcher narrows it:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;inspect_pr&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, { status: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;completed&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> });</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;apply_agents&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, { input: { verdict: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;update&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> } });</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;bash&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, { input: { command:</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">^</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">gh pr view</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }, count: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> });</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;read_file&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, {</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">  output</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: (</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">value</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=&gt;</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> String</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(value).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">includes</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;TODO&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">),</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p><code>input</code>, <code>output</code>, and <code>count</code> accept a literal, a <code>RegExp</code>, or a predicate. Object literals partial-deep-match, so <code>{ verdict: &quot;update&quot; }</code> matches arguments that also carry other keys. <code>status</code> is one of <code>completed</code>, <code>failed</code>, <code>pending</code>, or <code>rejected</code> (a human denied the approval). <code>calledSubagent</code> takes <code>{ output, status, count, callId }</code>. <code>event</code>, <code>notEvent</code>, and <code>eventOrder</code> take <code>{ data, count }</code>.</p><h3 id="grade-values-with-expectation-builders" tabindex="-1">Grade values with expectation builders <a class="header-anchor" href="#grade-values-with-expectation-builders" aria-label="Permalink to &quot;Grade values with expectation builders&quot;">​</a></h3><p><code>t.check(value, expectation)</code> grades any value: <code>t.reply</code>, a parsed JSON field, a tool&#39;s output.</p><table tabindex="0"><thead><tr><th>Builder</th><th>Checks</th><th>Severity</th></tr></thead><tbody><tr><td><code>includes(string | RegExp)</code></td><td>substring or match; structured values are stringified first</td><td>gate</td></tr><tr><td><code>equals(value)</code></td><td>deep equality</td><td>gate</td></tr><tr><td><code>matches(schema)</code></td><td>a Standard Schema (Zod, Valibot, ...) or anything with <code>safeParse</code></td><td>gate</td></tr><tr><td><code>similarity(expected)</code></td><td>normalized text similarity, 0-1</td><td>soft</td></tr><tr><td><code>satisfies(predicate, label)</code></td><td>your predicate; <code>label</code> is the failure detail</td><td>gate</td></tr></tbody></table><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { matches, satisfies } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/evals&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { z } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;zod&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> verdict</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> JSON</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">parse</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(t.reply </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">??</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;{}&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  verdict,</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">  matches</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(z.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">object</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({ ready: z.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">boolean</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(), blockers: z.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">array</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(z.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">string</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">()) }))</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  verdict.blockers.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">length</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">  satisfies</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">((</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">n</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=&gt;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> (n </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">as</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> number</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">&lt;=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> 3</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;at most 3 blockers&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><p><code>normalizedSimilarity(actual, expected)</code> returns the same 0-1 score as <code>similarity</code>, for use with <code>t.score</code>.</p><h3 id="record-without-gating" tabindex="-1">Record without gating <a class="header-anchor" href="#record-without-gating" aria-label="Permalink to &quot;Record without gating&quot;">​</a></h3><ul><li><code>t.metric(name, value)</code> records a structured score or label. It shows on the CLI result, the playground case card, JUnit output, and artifacts.</li><li><code>t.log(message)</code> records a debug line, streamed under <code>--verbose</code>.</li><li><code>t.skip(reason)</code> ends the case as skipped. Skipped cases report separately and never change the exit code. Call it before sending messages.</li></ul><h2 id="gates-soft-scores-and-verdicts" tabindex="-1">Gates, soft scores, and verdicts <a class="header-anchor" href="#gates-soft-scores-and-verdicts" aria-label="Permalink to &quot;Gates, soft scores, and verdicts&quot;">​</a></h2><p>Every assertion returns a handle, so severity rides on the assertion instead of a separate thresholds map:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(); </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// gate (default)</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;get_weather&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">soft</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(); </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// tracked, never fails</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(t.reply, </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">similarity</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Sunny, 72°F&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">atLeast</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.8</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">); </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// soft, with a bar</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.judge.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">closedQA</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;cites a source&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">gate</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.8</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">); </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// promoted to a gate</span></span></code></pre></div><ul><li><code>.gate(threshold?)</code> is hard. A miss fails the case and <code>eval</code> exits 1.</li><li><code>.soft(threshold?)</code> is tracked. With no threshold it never fails.</li><li><code>.atLeast(threshold)</code> is soft with a bar. A miss marks the case <code>scored</code>.</li></ul><p>Each case ends with one verdict:</p><table tabindex="0"><thead><tr><th>Verdict</th><th>Meaning</th><th>Exit code</th></tr></thead><tbody><tr><td><code>passed</code></td><td>every gate passed and no soft bar was missed</td><td>0</td></tr><tr><td><code>failed</code></td><td>a gate failed, or the test body threw</td><td>1</td></tr><tr><td><code>scored</code></td><td>only soft bars were missed</td><td>0, or 1 under <code>--strict</code></td></tr><tr><td><code>skipped</code></td><td><code>t.skip(reason)</code>, or a judge with no credentials</td><td>0</td></tr></tbody></table><p>The CLI prints soft misses as <code>~</code> and gate misses as <code>✗</code>. Start a new benchmark with <code>t.score(&quot;recall&quot;, recall)</code> and <code>.atLeast()</code> so its number reports for a while without blocking merges. Add <code>--strict</code> once the bars are trustworthy.</p><h2 id="judge-free-form-output" tabindex="-1">Judge free-form output <a class="header-anchor" href="#judge-free-form-output" aria-label="Permalink to &quot;Judge free-form output&quot;">​</a></h2><p>When wording matters and no regex captures it, <code>t.judge</code> grades with an LLM. The graders are <code>factuality(expected)</code>, <code>summarizes(expected)</code>, <code>closedQA(criteria)</code>, and <code>sql(expected)</code>. Each scores <code>t.reply</code> by default; pass <code>{ on }</code> to grade another value.</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> summary</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Why did CI fail on PR 42?&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.judge.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">factuality</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;The lint step failed on src/sidebar.ts.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">atLeast</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.7</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.judge.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">closedQA</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;names the failing step&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, { on: summary.message }).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">gate</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><p>Judge assertions are soft by default, so a judge never fails a build until you give it a bar with <code>.atLeast()</code> or promote it with <code>.gate()</code>. The recorded detail names the choice the judge made and its rationale.</p><p>The judge model comes from <code>defineEvalConfig({ judge })</code>, <code>defineEval({ judge })</code>, a case-level <code>judge</code>, or a per-call <code>{ model }</code>. The nearest one wins. A judge call with no model configured fails the case. A judge that cannot reach a model (no credential) ends the case as <code>skipped</code>, unless a deterministic gate already failed.</p><p>For a domain-specific judge whose verdict is not a single score, <code>t.judge.model(prompt)</code> sends a raw prompt to the same model and returns the reply. Record the parsed result with <code>t.score</code> or <code>t.check</code>. Anything derived from the agent under test is untrusted input to your prompt: wrap it with <code>fenceUntrusted</code> and include <code>EVAL_JUDGE_INJECTION_GUARD</code>, as the built-in graders do.</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> {</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  EVAL_JUDGE_INJECTION_GUARD,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  fenceUntrusted,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">} </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/evals&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> gold</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;XSS in search.ts&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;open redirect in login.ts&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">];</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> reply</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.judge.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">model</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  [</span></span>
+<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">    &quot;For each GOLD finding, answer whether SUBMISSION reports it.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">    &quot;Reply with one line per finding: &lt;index&gt; YES|NO.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">    EVAL_JUDGE_INJECTION_GUARD</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">    fenceUntrusted</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;GOLD&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, gold.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">map</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">((</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">g</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">i</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=&gt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> \`\${</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">i</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> +</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> 1</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">}. \${</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">g</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">}\`</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">join</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)),</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">    fenceUntrusted</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;SUBMISSION&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, t.reply </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">??</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">),</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ].</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">join</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\\n\\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> hits</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> reply.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">match</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">\\b</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">YES</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">\\b</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">g</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)?.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">length</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> ??</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> 0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">score</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;recall&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, hits </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">/</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> gold.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">length</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">atLeast</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.5</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><h2 id="keep-side-effects-out-of-eval-sessions" tabindex="-1">Keep side effects out of eval sessions <a class="header-anchor" href="#keep-side-effects-out-of-eval-sessions" aria-label="Permalink to &quot;Keep side effects out of eval sessions&quot;">​</a></h2><p>Eval sessions run the real agent, tools included. A reviewer that comments on GitHub or posts to Slack will do so from an eval unless the tool checks the session&#39;s purpose. Eval sessions carry <code>purpose: &quot;eval&quot;</code>; live traffic carries <code>&quot;live&quot;</code>. Branch on it in the tool, hook, or result handler that actuates:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// agent/tools/post_findings.ts</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">async </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">execute</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({ findings }, ctx) {</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">  if</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> (ctx.session.purpose </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">===</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;eval&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">    return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { posted: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">false</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, reason: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;eval&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, count: findings.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">length</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> };</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  }</span></span>
+<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">  // post the review</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Return a shaped result instead of throwing, so the eval can still assert <code>t.calledTool(&quot;post_findings&quot;, { input: ... })</code> on the decision. The same check belongs in <a href="./reference/hooks.html">hooks</a> that meter or page and in <code>defineResult</code> commits.</p><h2 id="worked-examples" tabindex="-1">Worked examples <a class="header-anchor" href="#worked-examples" aria-label="Permalink to &quot;Worked examples&quot;">​</a></h2><h3 id="multi-turn-grade-each-turn" tabindex="-1">Multi-turn: grade each turn <a class="header-anchor" href="#multi-turn-grade-each-turn" aria-label="Permalink to &quot;Multi-turn: grade each turn&quot;">​</a></h3><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/intro.eval.ts</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineEval, includes, satisfies } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/evals&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEval</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  description: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Introduces itself once; a repeat mention gets a short ack.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">  async</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">t</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">    const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> intro</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Meet Jenny! @Jenny introduce yourself.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    intro.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">expectOk</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(intro.message, </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">includes</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">jenny</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">i</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">));</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">    const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> repeat</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Meet, @Jenny!&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    repeat.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">usedNoTools</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      repeat.message,</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">      satisfies</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">((</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">r</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=&gt;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> (r </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">as</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> string</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">trim</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">().</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">length</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> 280</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;short ack&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    );</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      repeat.message,</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">      satisfies</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">((</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">r</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=&gt;</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> !</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">what i can do</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">i</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(r </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">as</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> string</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">), </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;no second intro&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    );</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p><code>t.succeeded()</code> grades the whole session. <code>repeat.usedNoTools()</code> and the checks on <code>repeat.message</code> read only the second turn, even though <code>t.reply</code> now holds its text.</p><h3 id="approvals-assert-the-parked-decision" tabindex="-1">Approvals: assert the parked decision <a class="header-anchor" href="#approvals-assert-the-parked-decision" aria-label="Permalink to &quot;Approvals: assert the parked decision&quot;">​</a></h3><p>For a tool with <code>needsApproval</code>, the turn parks instead of finishing. Gate on <code>t.parked()</code> and on the arguments the model chose:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/agents.eval.ts (one case; RULE and SLACK are fixture strings)</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">  id</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;update-rule&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">  description</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;A repeated billing rule parks the AGENTS.md write.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  async </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(t) {</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">    await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Weekly AGENTS.md review. Read week/ and call apply_agents once.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, {</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      workspaceFiles: {</span></span>
+<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">        &quot;week/prs.md&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">RULE</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">        &quot;week/slack.md&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">SLACK</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">        &quot;week/tree/AGENTS.md.txt&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;# API</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\\n\\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">Keep handlers thin.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    });</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">parked</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;apply_agents&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, { input: { verdict: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;update&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> } });</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">},</span></span></code></pre></div><p><code>t.parked()</code> and <code>t.succeeded()</code> are exclusive: a parked run is a clean stop on an unanswered approval, not a completed one. Pair the parked case with a sibling that expects <code>verdict: &quot;skip&quot;</code> and <code>t.succeeded()</code>, so both branches stay pinned.</p><h2 id="pin-fixtures" tabindex="-1">Pin fixtures <a class="header-anchor" href="#pin-fixtures" aria-label="Permalink to &quot;Pin fixtures&quot;">​</a></h2><p>A fixed input is what makes an eval repeatable. Pick the fixture by the surface under test.</p><table tabindex="0"><thead><tr><th>Agent surface</th><th>Fixture</th></tr></thead><tbody><tr><td>Chat or domain assistant</td><td>One canonical prompt string, chosen once and frozen</td></tr><tr><td>Tool-heavy</td><td>Run <code>agent-sdk call &lt;tool&gt;</code> first to pin what the tool returns, then freeze the prompt that triggers it</td></tr><tr><td>GitHub webhook</td><td><code>agent-sdk github replay &lt;pr&gt; --events &#39;*&#39; --dry-run --out fixtures/github</code> snapshots real payloads for offline replay (<a href="./guides/github.html#test-with-github-replay">GitHub guide</a>)</td></tr><tr><td>PR reviewer with host preparation</td><td>Diff, metadata, and gold labels pinned to commit SHAs; keep any live PR matrix small</td></tr><tr><td>Workspace-dependent</td><td><code>workspaceFiles</code> on the first <code>t.send</code>, never developer-machine paths</td></tr></tbody></table><p>Tag the fast, reliably passing core <code>smoke</code> and run <code>--tag smoke</code> in the inner loop. Leave slow or drift-prone cases untagged for explicit runs.</p><h3 id="materialize-api-backed-fixtures" tabindex="-1">Materialize API-backed fixtures <a class="header-anchor" href="#materialize-api-backed-fixtures" aria-label="Permalink to &quot;Materialize API-backed fixtures&quot;">​</a></h3><p>An input that only points at external data (a pull request URL, a snapshot id, a pair of commit SHAs) is not self-contained. Fetch it once and commit the rendered fixture before you expand the suite:</p><ol><li>Save the diff, metadata, and labels under <code>fixtures/</code> at pinned revisions.</li><li>Seed those files with <code>workspaceFiles</code>, or read them from the fixture directory.</li><li>Assert decisions and output shape against the saved evidence.</li><li>Keep a small <code>smoke</code> subset for any remaining live checks.</li></ol><p><code>maxConcurrency</code> limits parallel datapoints, not the model or API fan-out inside one datapoint. Materialized fixtures keep a large suite from exhausting provider and GitHub rate limits.</p><h3 id="load-a-dataset" tabindex="-1">Load a dataset <a class="header-anchor" href="#load-a-dataset" aria-label="Permalink to &quot;Load a dataset&quot;">​</a></h3><p>Read committed fixtures with <code>loadJson</code>, <code>loadJsonl</code>, and <code>loadYaml</code> from <code>@cursor/july/evals/loaders</code>. Relative paths resolve against the project root the runner discovered, not the cwd the CLI ran from. Eval files are ES modules, so top-level <code>await</code> can load a dataset and fan one file out over it:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/sql.eval.ts: sql/0000, sql/0001, ...</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineEval, equals } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/evals&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { loadYaml } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/evals/loaders&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> rows</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> loadYaml</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">&lt;{ </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">task</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">:</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> string</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">; </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">prompt</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">:</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> string</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">; </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">sql</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">:</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> string</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }[]&gt;(</span></span>
+<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">  &quot;evals/data/cases.yaml&quot;</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> rows.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">map</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">((</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">row</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=&gt;</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">  defineEval</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    description: row.task,</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">    async</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">t</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">      await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(row.prompt);</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(t.reply, </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">equals</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(row.sql));</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  })</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><h2 id="configure-eval-runs" tabindex="-1">Configure eval runs <a class="header-anchor" href="#configure-eval-runs" aria-label="Permalink to &quot;Configure eval runs&quot;">​</a></h2><p><code>evals/evals.config.ts</code> (or <code>.js</code>) holds project-wide defaults. It must set <code>maxConcurrency</code>. Each case issues real model requests, so concurrency is hard-capped at 200; the templates use 10. <code>eval --list</code> works without the file. Running a case does not.</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineEvalConfig } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/evals&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEvalConfig</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  maxConcurrency: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">20</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  timeoutMs: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">180_000</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  judge: { model: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;gpt-5.4-mini&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><table tabindex="0"><thead><tr><th>Option</th><th>Default</th><th>Meaning</th></tr></thead><tbody><tr><td><code>maxConcurrency</code></td><td>required</td><td>Datapoints in flight at once, 1-200</td></tr><tr><td><code>timeoutMs</code></td><td><code>180_000</code></td><td>Per-case timeout. Precedence: case or file <code>timeoutMs</code>, then <code>--timeout-ms</code>, then this value</td></tr><tr><td><code>judge</code></td><td>unset</td><td>Default judge model for <code>t.judge.*</code></td></tr><tr><td><code>reporters</code></td><td>unset</td><td>Destinations that observe every case; <code>--skip-report</code> suppresses them</td></tr><tr><td><code>maxPlaygroundRuns</code></td><td><code>20</code></td><td>Batches kept in playground history (cap 500). Counts <code>--prod</code> and <code>--url</code> batches, not the default local run</td></tr></tbody></table><p>Reporters ship results somewhere; the runner still does the grading. <code>JUnit({ filePath, suiteName? })</code> writes JUnit XML and <code>Artifacts({ dir })</code> writes per-case files, both from <code>@cursor/july/evals/reporters</code>. A custom reporter is an object with any of <code>onRunStart</code>, <code>onEvalComplete</code>, and <code>onRunComplete</code>. A reporter that throws is logged and never fails the run. CI usually attaches the built-in two with <code>--junit</code> and <code>--artifacts</code> instead of <code>reporters</code>, so output paths stay with the pipeline, not the eval author.</p><p>Playground batches survive restarts when the project configures durable storage. Otherwise they live in process memory until <code>serve</code> exits.</p><h2 id="run-evals-from-the-cli" tabindex="-1">Run evals from the CLI <a class="header-anchor" href="#run-evals-from-the-cli" aria-label="Permalink to &quot;Run evals from the CLI&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                          # discover only</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                                 # run all</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> builds/checkout</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                 # one datapoint</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> builds</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> search</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                   # several ids or prefixes</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> smoke</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> pull-request</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">  # any matching tag</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --verbose</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                       # t.log lines + reply snippets</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --timeout-ms</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> 300000</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">             # override the per-case timeout</span></span></code></pre></div><p>Id filters use OR semantics. Each filter selects an exact id and its descendants: <code>builds</code> selects <code>builds</code>, <code>builds/checkout</code>, and every other case below that path. Repeated tags also use OR. With both ids and tags, a case must match both groups.</p><p>By default <code>eval</code> boots a throwaway server with its own state root, so cases don&#39;t inherit your checkout&#39;s <code>AGENTS.md</code> and session state stays out of the project. Artifacts still land in the project state directory; see <a href="#where-results-land">Where results land</a>. <code>--slug</code> picks the target in a multi-agent directory.</p><p><code>--url</code> runs the batch on a running server instead, the same way <code>--prod</code> does: that server discovers its own <code>evals/</code>, results land in its playground history, and the local-only flags (<code>--junit</code>, <code>--artifacts</code>, <code>--max-concurrency</code>, <code>--skip-report</code>) do not apply. See <a href="#run-evals-in-the-playground-or-on-a-deployment">Run evals in the playground or on a deployment</a>.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --url</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> http://127.0.0.1:3000/weather-agent</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --bearer-token</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$AGENT_TOKEN</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;</span></span></code></pre></div><p>See <a href="./reference/cli.html#eval">CLI: eval</a> for every flag.</p><h3 id="credentials" tabindex="-1">Credentials <a class="header-anchor" href="#credentials" aria-label="Permalink to &quot;Credentials&quot;">​</a></h3><p>Model turns need a <a href="./reference/cli.html#environment-variables">Cursor credential</a>: <code>CURSOR_API_KEY</code>, <code>CURSOR_SERVICE_ACCOUNT_KEY</code>, or a saved <code>agent-sdk login</code>. The judge uses the same one. <code>eval --list</code> needs none.</p><h3 id="where-results-land" tabindex="-1">Where results land <a class="header-anchor" href="#where-results-land" aria-label="Permalink to &quot;Where results land&quot;">​</a></h3><p>Every local run writes artifacts to a timestamped directory under <code>evals/</code> in the project state directory, whatever <code>--state-root</code> says. <code>--artifacts &lt;dir&gt;</code> chooses the path and <code>--no-artifacts</code> skips them. The directory holds <code>summary.json</code>, <code>results.jsonl</code>, and <code>evals/&lt;case-id&gt;.json</code> with every assertion, the inputs, tool calls with arguments and output, the final text, and <code>t.log</code> lines. Start there when a case fails. <code>--out &lt;file&gt;</code> also writes the full results JSON to a path of your choice.</p><p>The artifact does not include the session&#39;s event stream. Pass <code>--state-root &lt;path&gt;</code> to keep the ephemeral server&#39;s <a href="./reference/sessions.html#where-does-the-agent-sdk-store-session-data">session data</a> on disk when you need the raw events.</p><h2 id="run-evals-in-ci" tabindex="-1">Run evals in CI <a class="header-anchor" href="#run-evals-in-ci" aria-label="Permalink to &quot;Run evals in CI&quot;">​</a></h2><p>Run the suite non-interactively, write JUnit for the CI annotations, and fail the job on a red gate:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># CURSOR_API_KEY comes from the CI secret store</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --no-stream</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --junit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> reports/evals.xml</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --artifacts</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> reports/evals</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">  &gt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> reports/evals.json</span></span></code></pre></div><p>The exit code follows the <a href="#gates-soft-scores-and-verdicts">verdict table</a>; <code>2</code> means nothing matched the selection. <code>--max-concurrency</code> overrides the project setting, for example to run lower on a shared runner.</p><p>The JSON on stdout carries the totals and one result per case:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;ok&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;passed&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;failed&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;scored&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;skipped&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;strict&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">false</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;artifactsDir&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;/work/my-agent/reports/evals&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;results&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    {</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;id&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;readiness&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;verdict&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;passed&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;ok&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;assertions&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">        { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;succeeded&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;passed&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">        { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;calledTool(inspect_pr)&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;passed&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">      ],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;sessionId&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;ses_123&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;inputs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Is https://github.com/acme/checkout/pull/42 ready to approve?&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;toolCalls&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;toolName&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;inspect_pr&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;isError&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">false</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;metrics&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {},</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;logs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [],</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;durationMs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">12340</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    }</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ]</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Each result can also include <code>description</code>, <code>finalText</code>, <code>tools</code>, <code>error</code>, <code>skipReason</code>, <code>metadata</code>, <code>tags</code>, and tool <code>args</code> and <code>output</code>. A soft miss shows as <code>&quot;severity&quot;: &quot;soft&quot;</code> with <code>score</code> and <code>threshold</code> on the assertion. This shape lets CI report the failed assertion without parsing terminal text.</p><p>Keep CI green without weakening gates:</p><ul><li>Run <code>--tag smoke</code> on every push and the full suite on a schedule.</li><li>For probabilistic behavior, use <code>iterations</code> and a soft bar instead of one hard gate.</li></ul><h2 id="run-evals-in-the-playground-or-on-a-deployment" tabindex="-1">Run evals in the playground or on a deployment <a class="header-anchor" href="#run-evals-in-the-playground-or-on-a-deployment" aria-label="Permalink to &quot;Run evals in the playground or on a deployment&quot;">​</a></h2><p>Start the server, open the playground, and choose <strong>Evals</strong>. Run every case or one case, watch progress, and open the resulting session trace. Playground runs target the live server instead of an ephemeral one, so their sessions appear in the session list. One batch runs at a time.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> serve</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span></span></code></pre></div><p><code>--prod</code> (or <code>--url</code>) starts the same server-side batch on the team&#39;s hosted deployment (or the server you name), so results land in that server&#39;s playground history:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> deepsec</span></span>
+<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Eval ID: &lt;evalId&gt;</span></span>
+<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Playground: &lt;deployment&gt;/playground?view=evals&amp;evalRunId=&lt;evalId&gt;</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> status</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">evalI</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">d</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">&gt;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> cancel</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">evalI</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">d</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">&gt;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span></span></code></pre></div><p>The CLI prints the Eval ID as soon as the batch is accepted. Pass <code>--no-wait</code> to return right away and poll with <code>eval status</code> later; it exits <code>3</code> while the batch is still running. Hosted history follows <code>maxPlaygroundRuns</code> and the persistence rule under <a href="#configure-eval-runs">Configure eval runs</a>. The HTTP surface is under <a href="./reference/http-api.html#playground-eval-routes">Playground eval routes</a>.</p><h2 id="keep-improvements-with-regression-evals" tabindex="-1">Keep improvements with regression evals <a class="header-anchor" href="#keep-improvements-with-regression-evals" aria-label="Permalink to &quot;Keep improvements with regression evals&quot;">​</a></h2><p>Every <a href="./hillclimbing.html">hillclimb</a> round that keeps a change must land an eval that would have failed before the change. If you can&#39;t express the improvement as a gate (a <code>calledTool</code> shift, a bounded <code>maxToolCalls</code>, an output-shape check), the improvement is unverified, and it&#39;ll regress silently.</p><p>The rule cuts the other way too: never weaken an existing gate to make a round pass. That&#39;s the freeze line moving, and it turns your regression suite into a list of checks that no longer protect anything.</p><h2 id="what-s-next" tabindex="-1">What&#39;s next <a class="header-anchor" href="#what-s-next" aria-label="Permalink to &quot;What&#39;s next&quot;">​</a></h2><p>Continue with these pages:</p><ul><li><a href="./hillclimbing.html">Hillclimbing</a>: the loop evals make trustworthy</li><li><a href="./building-with-agents.html">Building agents with agents</a>: have a coding agent write the first suite</li><li><a href="./guides/github.html">GitHub guide</a>: deterministic webhook fixtures with <code>github replay</code></li><li><a href="./reference/sessions.html">Sessions and streaming</a>: the events <code>t.events</code> contains</li></ul>`,131)])])}const E=i(n,[["render",h]]);export{c as __pageData,E as default};

package/dist/docs/assets/evals.md.BLDRt5LH.lean.js

@@ -0,0 +1 @@
+import{_ as i,c as a,o as e,a3 as t}from"./chunks/framework.dypDpWZ3.js";const c=JSON.parse(`{"title":"Evals","description":"Write defineEval cases that gate an agent's trajectory, run them with agent-sdk eval locally and in CI, and lock every kept improvement with a regression check.","frontmatter":{"title":"Evals","description":"Write defineEval cases that gate an agent's trajectory, run them with agent-sdk eval locally and in CI, and lock every kept improvement with a regression check."},"headers":[],"relativePath":"evals.md","filePath":"evals.md"}`),n={name:"evals.md"};function h(l,s,p,d,k,r){return e(),a("div",null,[...s[0]||(s[0]=[t("",131)])])}const E=i(n,[["render",h]]);export{c as __pageData,E as default};

package/dist/docs/assets/guides_agent-to-agent.md.BI0xclmy.js

@@ -0,0 +1,14 @@
+import{_ as s,c as t,o as a,a3 as n}from"./chunks/framework.dypDpWZ3.js";const g=JSON.parse('{"title":"Agent-to-agent","description":"Wire two Agent SDK agents on one host so one can ask the other.","frontmatter":{"title":"Agent-to-agent","description":"Wire two Agent SDK agents on one host so one can ask the other."},"headers":[],"relativePath":"guides/agent-to-agent.md","filePath":"guides/agent-to-agent.md"}'),i={name:"guides/agent-to-agent.md"};function o(h,e,p,l,r,d){return a(),t("div",null,[...e[0]||(e[0]=[n(`<h1 id="agent-to-agent" tabindex="-1">Agent-to-agent <a class="header-anchor" href="#agent-to-agent" aria-label="Permalink to &quot;Agent-to-agent&quot;">​</a></h1><p>Point one agent at another on the same host. The peer keeps its own session, instructions, and tools.</p><h2 id="peer-or-subagent" tabindex="-1">Peer or subagent? <a class="header-anchor" href="#peer-or-subagent" aria-label="Permalink to &quot;Peer or subagent?&quot;">​</a></h2><p>Use a <a href="./../reference/subagents.html">subagent</a> to split one job into roles inside a single agent. Use a peer when the specialist is a full agent you would run on its own.</p><table tabindex="0"><thead><tr><th></th><th>Subagent (<code>agent/subagents/&lt;id&gt;/</code>)</th><th>Peer (<code>defineConnection({ agent })</code>)</th></tr></thead><tbody><tr><td>Runs as</td><td>a child of this agent</td><td>an agent on the same host</td></tr><tr><td>Tools, MCP connections, sessions</td><td>the parent&#39;s</td><td>its own</td></tr><tr><td>Who can reach it</td><td>only the parent</td><td>other agents, MCP clients, its playground</td></tr></tbody></table><h2 id="how-do-i-wire-two-agents" tabindex="-1">How do I wire two agents? <a class="header-anchor" href="#how-do-i-wire-two-agents" aria-label="Permalink to &quot;How do I wire two agents?&quot;">​</a></h2><p>Put both projects in one directory:</p><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>agents/</span></span>
+<span class="line"><span>  concierge/</span></span>
+<span class="line"><span>    agent/instructions.md</span></span>
+<span class="line"><span>    agent/mcp-connections/weather.ts</span></span>
+<span class="line"><span>  weather-agent/</span></span>
+<span class="line"><span>    agent/instructions.md</span></span></code></pre></div><p><code>weather-agent</code> is a normal Agent SDK project. On the concierge, add a connection. Set <code>agent</code> to the peer&#39;s slug, which is its directory name:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// agents/concierge/agent/mcp-connections/weather.ts</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineConnection } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/connections&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineConnection</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  agent: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;weather-agent&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  description: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Delegate weather questions to the weather agent.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p>The filename is the MCP server name. The concierge model sees the peer under <code>weather</code>. See <a href="./../reference/http-api.html#mcp-endpoint">HTTP API</a> for the MCP tools.</p><p>In the concierge <code>instructions.md</code>, say when to ask: &quot;When a request needs weather, ask the <code>weather</code> peer.&quot; Scope the line to the work the peer should handle. If two agents can ask each other, they will loop.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> ./agents</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> chat</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --url</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> http://127.0.0.1:3000/concierge</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --message</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;What&#39;s the weather in Paris right now?&quot;</span></span></code></pre></div><p>Cursor-managed hosting deploys one agent per slug. Keep peers on one self-hosted process. Cloud-runtime turns need <code>--public-url</code> and <code>--bearer-token</code>. See <a href="./../deployment.html">Deployment</a>.</p><p>Unknown slugs and self-references fail <code>serve</code> at startup. Peers need the default multi-agent layout.</p><h2 id="what-s-next" tabindex="-1">What&#39;s next <a class="header-anchor" href="#what-s-next" aria-label="Permalink to &quot;What&#39;s next&quot;">​</a></h2><ul><li><a href="./../reference/connections.html#peer-mcp-connection">MCP connections</a>: <code>{ agent }</code></li><li><a href="./../reference/subagents.html">Subagents</a>: specialists inside one agent</li><li><a href="./../reference/http-api.html#mcp-endpoint">HTTP API</a>: <code>/v1/mcp</code></li></ul>`,17)])])}const k=s(i,[["render",o]]);export{g as __pageData,k as default};

package/dist/docs/assets/guides_agent-to-agent.md.BI0xclmy.lean.js

@@ -0,0 +1 @@
+import{_ as s,c as t,o as a,a3 as n}from"./chunks/framework.dypDpWZ3.js";const g=JSON.parse('{"title":"Agent-to-agent","description":"Wire two Agent SDK agents on one host so one can ask the other.","frontmatter":{"title":"Agent-to-agent","description":"Wire two Agent SDK agents on one host so one can ask the other."},"headers":[],"relativePath":"guides/agent-to-agent.md","filePath":"guides/agent-to-agent.md"}'),i={name:"guides/agent-to-agent.md"};function o(h,e,p,l,r,d){return a(),t("div",null,[...e[0]||(e[0]=[n("",17)])])}const k=s(i,[["render",o]]);export{g as __pageData,k as default};