@cursor/july 0.1.29 → 0.1.31

package/dist/docs/assets/{evals.md.DYOjkRCX.js → evals.md.BIUoVZ6X.js}

@@ -1,4 +1,4 @@
-import{_ as i,c as a,o as e,ag as t}from"./chunks/framework.CAZyNGu9.js";const c=JSON.parse('{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agentkit eval, and use them as regression checks.","frontmatter":{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agentkit eval, and use them as regression checks."},"headers":[],"relativePath":"evals.md","filePath":"evals.md"}'),n={name:"evals.md"};function l(h,s,p,d,o,r){return e(),a("div",null,[...s[0]||(s[0]=[t(`<h1 id="evals" tabindex="-1">Evals <a class="header-anchor" href="#evals" aria-label="Permalink to &quot;Evals&quot;">​</a></h1><p>An eval is a repeatable check that runs your agent against a fixed input and gates the recorded trajectory: the run completed, the right tool ran, the reply has the right shape. Evals are how you know a prompt tweak helped, a refactor didn&#39;t regress the agent, and last month&#39;s fix is still holding.</p><p>Evals exercise the same surface your users hit. The runner starts (or targets) a real agent server, drives sessions over the public API, and grades what comes back. A passing eval means the agent started, accepted a message, and did what you asserted.</p><div class="note custom-block github-alert"><p class="custom-block-title">NOTE</p><p>Import paths here use <code>@cursor/july/evals</code>. On projects still using <code>@cursor/july</code>, swap the import and run <code>agent-serve eval</code>. See <a href="./README.html#run-the-cli">Run the CLI</a> for the full rename table.</p></div><h2 id="define-evals-with-defineeval" tabindex="-1">Define evals with <code>defineEval</code> <a class="header-anchor" href="#define-evals-with-defineeval" aria-label="Permalink to &quot;Define evals with \`defineEval\`&quot;">​</a></h2><p>agentkit discovers evals under the project-root <code>evals/</code> directory, in <code>.eval.ts</code> or <code>.eval.js</code> files. That&#39;s a sibling of <code>agent/</code>, never inside it (<code>agent/evals/</code> is silently ignored). TypeScript is the normal authoring format.</p><p>The file path is the eval&#39;s identity, so you don&#39;t author an id. Directories group related evals: <code>evals/builds/api.eval.ts</code> becomes id <code>builds/api</code>. An <code>index</code> filename collapses to its directory, so <code>evals/builds/index.eval.ts</code> becomes <code>builds</code>.</p><p>An eval is a single <code>async test(t)</code>. You drive the agent with <code>t</code> and assert on the run with the same <code>t</code>:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/readiness.eval.ts</span></span>
+import{_ as i,c as a,o as e,ag as t}from"./chunks/framework.CAZyNGu9.js";const c=JSON.parse('{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agent-sdk eval, and use them as regression checks.","frontmatter":{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agent-sdk eval, and use them as regression checks."},"headers":[],"relativePath":"evals.md","filePath":"evals.md"}'),n={name:"evals.md"};function l(h,s,p,d,o,r){return e(),a("div",null,[...s[0]||(s[0]=[t(`<h1 id="evals" tabindex="-1">Evals <a class="header-anchor" href="#evals" aria-label="Permalink to &quot;Evals&quot;">​</a></h1><p>An eval is a repeatable check that runs your agent against a fixed input and gates the recorded trajectory: the run completed, the right tool ran, the reply has the right shape. Evals are how you know a prompt tweak helped, a refactor didn&#39;t regress the agent, and last month&#39;s fix is still holding.</p><p>Evals exercise the same surface your users hit. The runner starts (or targets) a real agent server, drives sessions over the public API, and grades what comes back. A passing eval means the agent started, accepted a message, and did what you asserted.</p><div class="note custom-block github-alert"><p class="custom-block-title">NOTE</p><p>Import paths here use <code>@cursor/july/evals</code>. On projects still using <code>@cursor/july</code>, swap the import and run <code>agent-serve eval</code>. See <a href="./README.html#run-the-cli">Run the CLI</a> for the full rename table.</p></div><h2 id="define-evals-with-defineeval" tabindex="-1">Define evals with <code>defineEval</code> <a class="header-anchor" href="#define-evals-with-defineeval" aria-label="Permalink to &quot;Define evals with \`defineEval\`&quot;">​</a></h2><p>The Agent SDK discovers evals under the project-root <code>evals/</code> directory, in <code>.eval.ts</code> or <code>.eval.js</code> files. That&#39;s a sibling of <code>agent/</code>, never inside it (<code>agent/evals/</code> is silently ignored). TypeScript is the normal authoring format.</p><p>The file path is the eval&#39;s identity, so you don&#39;t author an id. Directories group related evals: <code>evals/builds/api.eval.ts</code> becomes id <code>builds/api</code>. An <code>index</code> filename collapses to its directory, so <code>evals/builds/index.eval.ts</code> becomes <code>builds</code>.</p><p>An eval is a single <code>async test(t)</code>. You drive the agent with <code>t</code> and assert on the run with the same <code>t</code>:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/readiness.eval.ts</span></span>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineEval, includes } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/evals&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
 <span class="line"></span>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEval</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
@@ -58,15 +58,15 @@ import{_ as i,c as a,o as e,ag as t}from"./chunks/framework.CAZyNGu9.js";const c
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  toolResults.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">length</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">  satisfies</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">((</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">n</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=&gt;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> (n </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">as</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> number</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">&lt;=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> 4</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;at most 4 tool calls&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><p>A case with no explicit gates falls back to whether at least one turn completed successfully. Add <code>t.succeeded()</code> and behavior-specific gates anyway. They make the contract visible during review.</p><h2 id="run-evals-from-the-cli" tabindex="-1">Run evals from the CLI <a class="header-anchor" href="#run-evals-from-the-cli" aria-label="Permalink to &quot;Run evals from the CLI&quot;">​</a></h2><p>The <code>eval</code> command discovers, filters, and runs cases.</p><p>Run the CLI under Node 22.13 or newer. Do not use Bun. Its HTTP/2 client breaks tool-result streams and causes eval turns to fail.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                         # discover only</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                                # run all</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> builds/checkout</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                # one datapoint</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> builds</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> search</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                  # several ids or prefixes</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> smoke</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> pull-request</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # any matching tag</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --no-stream</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">             # machine-readable results</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --verbose</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                      # logs + reply snippets</span></span></code></pre></div><p>Id filters use OR semantics. Each filter selects an exact id and its descendants. For example, <code>builds</code> selects <code>builds</code>, <code>builds/checkout</code>, and every other case below that path. Repeated tags also use OR semantics. When you provide both ids and tags, a case must match both groups.</p><p><code>eval</code> boots an ephemeral server on port 0 with a temp state root outside the project, so cases don&#39;t inherit ambient monorepo rules and don&#39;t pollute <code>.agentkit/</code>. Point <code>--url</code> at a running server to eval a live agent instead:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><p>A case with no explicit gates falls back to whether at least one turn completed successfully. Add <code>t.succeeded()</code> and behavior-specific gates anyway. They make the contract visible during review.</p><h2 id="run-evals-from-the-cli" tabindex="-1">Run evals from the CLI <a class="header-anchor" href="#run-evals-from-the-cli" aria-label="Permalink to &quot;Run evals from the CLI&quot;">​</a></h2><p>The <code>eval</code> command discovers, filters, and runs cases.</p><p>Run the CLI under Node 22.13 or newer. Do not use Bun. Its HTTP/2 client breaks tool-result streams and causes eval turns to fail.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                         # discover only</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                                # run all</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> builds/checkout</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                # one datapoint</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> builds</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> search</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                  # several ids or prefixes</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> smoke</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> pull-request</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # any matching tag</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --no-stream</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">             # machine-readable results</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --verbose</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                      # logs + reply snippets</span></span></code></pre></div><p>Id filters use OR semantics. Each filter selects an exact id and its descendants. For example, <code>builds</code> selects <code>builds</code>, <code>builds/checkout</code>, and every other case below that path. Repeated tags also use OR semantics. When you provide both ids and tags, a case must match both groups.</p><p><code>eval</code> boots an ephemeral server on port 0 with a temp state root outside the project, so cases don&#39;t inherit ambient monorepo rules and don&#39;t pollute <code>.agent-sdk/</code>. Point <code>--url</code> at a running server to eval a live agent instead:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --url</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> http://127.0.0.1:3000/weather-agent</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
-<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --bearer-token</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$AGENT_TOKEN</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;</span></span></code></pre></div><p>The eval definitions still come from <code>--dir</code>; <code>--url</code> only changes the agent that receives the turns. For a locally mounted multi-agent directory, <code>--slug weather-agent</code> chooses the target. Use <code>--state-root</code> to keep ephemeral session state at a chosen path, <code>--timeout-ms</code> to override the project timeout, and <code>--no-stream</code> to keep live progress off stderr. A TTY streams turn progress by default. <code>--verbose</code> still writes <code>t.log</code> lines to stderr and adds reply snippets to text results.</p><p>Model turns need a Cursor credential from <code>agentkit login</code> or <code>CURSOR_API_KEY</code>.</p><p>The exit code is <code>0</code> when every selected case passes, <code>1</code> when any case fails, and <code>2</code> when no case matches. <code>--list</code> exits <code>0</code>, including when it finds no cases.</p><p>For a compact command index, see <a href="./reference/cli.html#eval">CLI: eval</a>.</p><h3 id="json-results" tabindex="-1">JSON results <a class="header-anchor" href="#json-results" aria-label="Permalink to &quot;JSON results&quot;">​</a></h3><p>Use <code>--json --no-stream</code> in scripts and CI. The top-level result carries the totals and one result per case:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --bearer-token</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$AGENT_TOKEN</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;</span></span></code></pre></div><p>The eval definitions still come from <code>--dir</code>; <code>--url</code> only changes the agent that receives the turns. For a locally mounted multi-agent directory, <code>--slug weather-agent</code> chooses the target. Use <code>--state-root</code> to keep ephemeral session state at a chosen path, <code>--timeout-ms</code> to override the project timeout, and <code>--no-stream</code> to keep live progress off stderr. A TTY streams turn progress by default. <code>--verbose</code> still writes <code>t.log</code> lines to stderr and adds reply snippets to text results.</p><p>Model turns need a Cursor credential from <code>agent-sdk login</code> or <code>CURSOR_API_KEY</code>.</p><p>The exit code is <code>0</code> when every selected case passes, <code>1</code> when any case fails, and <code>2</code> when no case matches. <code>--list</code> exits <code>0</code>, including when it finds no cases.</p><p>For a compact command index, see <a href="./reference/cli.html#eval">CLI: eval</a>.</p><h3 id="json-results" tabindex="-1">JSON results <a class="header-anchor" href="#json-results" aria-label="Permalink to &quot;JSON results&quot;">​</a></h3><p>Use <code>--json --no-stream</code> in scripts and CI. The top-level result carries the totals and one result per case:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;ok&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;passed&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;failed&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
@@ -82,10 +82,10 @@ import{_ as i,c as a,o as e,ag as t}from"./chunks/framework.CAZyNGu9.js";const c
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;durationMs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">12340</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    }</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ]</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Each case result can also include <code>description</code>, <code>finalText</code>, <code>tools</code>, <code>error</code>, and tool arguments or output. This shape lets CI report the failed assertion without parsing terminal text.</p><h2 id="run-evals-in-the-playground" tabindex="-1">Run evals in the playground <a class="header-anchor" href="#run-evals-in-the-playground" aria-label="Permalink to &quot;Run evals in the playground&quot;">​</a></h2><p>Start the server with <code>--dev</code>, open the playground, and choose <strong>Evals</strong>. You can run every case or one case, watch progress, and open the resulting session trace.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> serve</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dev</span></span></code></pre></div><p>Playground runs target the live server instead of an ephemeral one. Their sessions appear in the session list. One eval batch can run at a time. By default those batches are <strong>in-memory only</strong> (capped by <code>maxPlaygroundRuns</code>); set <code>persistRuns</code> in <code>evals.config.ts</code> if you need them after a serve restart — see <a href="#configure-eval-runs">Configure eval runs</a>.</p><p>The UI uses the playground eval routes (available without <code>--dev</code>): <code>GET /v1/dev/evals</code> lists datapoints and config (includes <code>maxPlaygroundRuns</code> / <code>durableRuns</code>), <code>GET /v1/dev/evals/runs</code> rehydrates recent batches after navigation, <code>POST /v1/dev/evals/runs</code> starts a batch (returns an <strong>Eval ID</strong> / <code>runId</code>), <code>GET /v1/dev/evals/runs/:runId</code> polls it, and <code>POST /v1/dev/evals/runs/:runId/cancel</code> cancels a running batch. See <a href="./reference/http-api.html#playground-eval-routes">Playground eval routes</a>. The start request returns <code>202</code> while cases run in the background. Poll until the snapshot status becomes <code>completed</code>, <code>failed</code>, or <code>cancelled</code>. Configuration errors appear on a failed snapshot.</p><p>On <code>--prod</code> / <code>--url</code>, the CLI prints the Eval ID as soon as the batch is accepted (and a Playground deep link with <code>?view=evals&amp;evalRunId=…</code>):</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> deepsec</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Each case result can also include <code>description</code>, <code>finalText</code>, <code>tools</code>, <code>error</code>, and tool arguments or output. This shape lets CI report the failed assertion without parsing terminal text.</p><h2 id="run-evals-in-the-playground" tabindex="-1">Run evals in the playground <a class="header-anchor" href="#run-evals-in-the-playground" aria-label="Permalink to &quot;Run evals in the playground&quot;">​</a></h2><p>Start the server with <code>--dev</code>, open the playground, and choose <strong>Evals</strong>. You can run every case or one case, watch progress, and open the resulting session trace.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> serve</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dev</span></span></code></pre></div><p>Playground runs target the live server instead of an ephemeral one. Their sessions appear in the session list. One eval batch can run at a time. By default those batches are <strong>in-memory only</strong> (capped by <code>maxPlaygroundRuns</code>); set <code>persistRuns</code> in <code>evals.config.ts</code> if you need them after a serve restart — see <a href="#configure-eval-runs">Configure eval runs</a>.</p><p>The UI uses the playground eval routes (available without <code>--dev</code>): <code>GET /v1/dev/evals</code> lists datapoints and config (includes <code>maxPlaygroundRuns</code> / <code>durableRuns</code>), <code>GET /v1/dev/evals/runs</code> rehydrates recent batches after navigation, <code>POST /v1/dev/evals/runs</code> starts a batch (returns an <strong>Eval ID</strong> / <code>runId</code>), <code>GET /v1/dev/evals/runs/:runId</code> polls it, and <code>POST /v1/dev/evals/runs/:runId/cancel</code> cancels a running batch. See <a href="./reference/http-api.html#playground-eval-routes">Playground eval routes</a>. The start request returns <code>202</code> while cases run in the background. Poll until the snapshot status becomes <code>completed</code>, <code>failed</code>, or <code>cancelled</code>. Configuration errors appear on a failed snapshot.</p><p>On <code>--prod</code> / <code>--url</code>, the CLI prints the Eval ID as soon as the batch is accepted (and a Playground deep link with <code>?view=evals&amp;evalRunId=…</code>):</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> deepsec</span></span>
 <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Eval ID: evalrun_…</span></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Cancel:  agentkit eval cancel evalrun_… --prod --slug vulnerability-scanner</span></span>
+<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Cancel:  agent-sdk eval cancel evalrun_… --prod --slug vulnerability-scanner</span></span>
 <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Playground: https://…/playground?view=evals&amp;evalRunId=evalrun_…</span></span>
 <span class="line"></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> cancel</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> evalrun_…</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> status</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> evalrun_…</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span></span></code></pre></div><p>The Evals tab prefers the server’s in-flight batch (<code>activeRunId</code>) over a stale tab-local remembered id, so CLI / Slack kicks show up without an incognito window.</p><h2 id="what-good-cases-assert" tabindex="-1">What good cases assert <a class="header-anchor" href="#what-good-cases-assert" aria-label="Permalink to &quot;What good cases assert&quot;">​</a></h2><p>Gate decisions and shape, not prose. Model wording varies run to run. Tool choice, tool avoidance, and output structure are the stable contract.</p><ol><li><code>t.succeeded()</code>: always, first.</li><li>The tool decision: <code>calledTool</code> for the intended path, <code>notCalledTool</code> for the likely wrong alternative. The pair is stronger than either alone.</li><li>Output shape: a regex for the contract (<code>/ready|blocked/i</code>, a JSON marker, a findings-block fence), never exact sentences.</li><li>For structured output, parse <code>t.reply</code> and check fields with <code>satisfies</code> instead of substring-matching JSON.</li></ol><p>The common failure modes: asserting exact phrasing, packing more than about five gates into one case (split it), and cases that depend on live external state that drifts (pin the input; see fixtures).</p><h2 id="pick-fixtures-by-agent-type" tabindex="-1">Pick fixtures by agent type <a class="header-anchor" href="#pick-fixtures-by-agent-type" aria-label="Permalink to &quot;Pick fixtures by agent type&quot;">​</a></h2><p>The right fixture depends on the surface under test.</p><table tabindex="0"><thead><tr><th>Agent surface</th><th>Fixture</th></tr></thead><tbody><tr><td>Chat / domain assistant</td><td>A canonical prompt string, chosen once and frozen</td></tr><tr><td>Tool-heavy</td><td>Run <code>agentkit call &lt;tool&gt;</code> first to pin what the tool returns, then freeze the prompt that triggers it</td></tr><tr><td>GitHub webhook</td><td><code>agentkit github replay &lt;pr&gt; --events &#39;*&#39; --dry-run --out fixtures/github</code> snapshots real payloads for offline replay (<a href="./guides/github.html">GitHub guide</a>)</td></tr><tr><td>PR reviewer with host preparation</td><td>Diff, metadata, and gold labels pinned to commit SHAs; keep any live PR matrix small</td></tr><tr><td>Workspace-dependent</td><td><code>workspaceFiles</code> in <code>t.send</code> options, never developer-machine paths</td></tr></tbody></table><p>Tag the fast, reliably passing core <code>smoke</code> and run <code>--tag smoke</code> in the inner loop. Leave slow or flaky-prone cases untagged for explicit runs.</p><h3 id="materialize-api-backed-fixtures" tabindex="-1">Materialize API-backed fixtures <a class="header-anchor" href="#materialize-api-backed-fixtures" aria-label="Permalink to &quot;Materialize API-backed fixtures&quot;">​</a></h3><p>An input that only points at external data, such as a pull request URL, snapshot id, or pair of commit SHAs, is not self-contained. Fetch it once and commit the rendered fixture before you expand the suite.</p><ol><li>Save the diff, metadata, and labels under <code>fixtures/</code> at pinned revisions.</li><li>Seed those files with <code>workspaceFiles</code>, or read them from the fixture directory.</li><li>Assert decisions and output shape against the saved evidence.</li><li>Keep a small <code>smoke</code> subset for any remaining live pipeline checks.</li></ol><p><code>maxConcurrency</code> limits parallel datapoints. It does not limit model or API fan-out inside one datapoint. Materialized fixtures prevent a large suite from exhausting provider and GitHub rate limits. The <a href="./../skills/evals/SKILL.html">evals skill</a> has the full fixture workflow.</p><h2 id="keep-improvements-with-regression-evals" tabindex="-1">Keep improvements with regression evals <a class="header-anchor" href="#keep-improvements-with-regression-evals" aria-label="Permalink to &quot;Keep improvements with regression evals&quot;">​</a></h2><p>Every <a href="./hillclimbing.html">hillclimb</a> round that keeps a change must land an eval that would have failed before the change. If you can&#39;t express the improvement as a gate (a <code>calledTool</code> shift, a bounded <code>action.result</code> count, an output-shape regex), the improvement is unverified, and it&#39;ll regress silently.</p><p>The rule cuts the other way too: never weaken an existing gate to make a round pass. That&#39;s the freeze line moving, and it turns your regression suite into a list of checks that no longer protect anything.</p><h2 id="compare-variants-on-live-traffic" tabindex="-1">Compare variants on live traffic <a class="header-anchor" href="#compare-variants-on-live-traffic" aria-label="Permalink to &quot;Compare variants on live traffic&quot;">​</a></h2><p>Use <code>defineAB</code> to compare variant metrics on live sessions. It is not a test runner and has no <code>agentkit ab</code> command. Keep <code>defineEval</code> as the regression ratchet. Eval sessions do not enroll or change live metrics. See <a href="./ab.html">Live A/B metrics</a> for assignment, behavior, collection, and inspection.</p><h2 id="what-s-next" tabindex="-1">What&#39;s next <a class="header-anchor" href="#what-s-next" aria-label="Permalink to &quot;What&#39;s next&quot;">​</a></h2><p>Continue with these pages:</p><ul><li><a href="./ab.html">Live A/B metrics</a>: sticky variants and cumulative metrics on live sessions</li><li><a href="./hillclimbing.html">Hillclimbing</a>: the loop evals make trustworthy</li><li><a href="./building-with-agents.html">Building agents with agents</a>: have a coding agent write the first suite</li><li><a href="./guides/github.html">GitHub guide</a>: deterministic webhook fixtures with <code>github replay</code></li><li><a href="./reference/sessions.html">Sessions and streaming</a>: the events <code>t.events</code> contains</li></ul>`,77)])])}const g=i(n,[["render",l]]);export{c as __pageData,g as default};
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> cancel</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> evalrun_…</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> status</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> evalrun_…</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span></span></code></pre></div><p>The Evals tab prefers the server’s in-flight batch (<code>activeRunId</code>) over a stale tab-local remembered id, so CLI / Slack kicks show up without an incognito window.</p><h2 id="what-good-cases-assert" tabindex="-1">What good cases assert <a class="header-anchor" href="#what-good-cases-assert" aria-label="Permalink to &quot;What good cases assert&quot;">​</a></h2><p>Gate decisions and shape, not prose. Model wording varies run to run. Tool choice, tool avoidance, and output structure are the stable contract.</p><ol><li><code>t.succeeded()</code>: always, first.</li><li>The tool decision: <code>calledTool</code> for the intended path, <code>notCalledTool</code> for the likely wrong alternative. The pair is stronger than either alone.</li><li>Output shape: a regex for the contract (<code>/ready|blocked/i</code>, a JSON marker, a findings-block fence), never exact sentences.</li><li>For structured output, parse <code>t.reply</code> and check fields with <code>satisfies</code> instead of substring-matching JSON.</li></ol><p>The common failure modes: asserting exact phrasing, packing more than about five gates into one case (split it), and cases that depend on live external state that drifts (pin the input; see fixtures).</p><h2 id="pick-fixtures-by-agent-type" tabindex="-1">Pick fixtures by agent type <a class="header-anchor" href="#pick-fixtures-by-agent-type" aria-label="Permalink to &quot;Pick fixtures by agent type&quot;">​</a></h2><p>The right fixture depends on the surface under test.</p><table tabindex="0"><thead><tr><th>Agent surface</th><th>Fixture</th></tr></thead><tbody><tr><td>Chat / domain assistant</td><td>A canonical prompt string, chosen once and frozen</td></tr><tr><td>Tool-heavy</td><td>Run <code>agent-sdk call &lt;tool&gt;</code> first to pin what the tool returns, then freeze the prompt that triggers it</td></tr><tr><td>GitHub webhook</td><td><code>agent-sdk github replay &lt;pr&gt; --events &#39;*&#39; --dry-run --out fixtures/github</code> snapshots real payloads for offline replay (<a href="./guides/github.html">GitHub guide</a>)</td></tr><tr><td>PR reviewer with host preparation</td><td>Diff, metadata, and gold labels pinned to commit SHAs; keep any live PR matrix small</td></tr><tr><td>Workspace-dependent</td><td><code>workspaceFiles</code> in <code>t.send</code> options, never developer-machine paths</td></tr></tbody></table><p>Tag the fast, reliably passing core <code>smoke</code> and run <code>--tag smoke</code> in the inner loop. Leave slow or flaky-prone cases untagged for explicit runs.</p><h3 id="materialize-api-backed-fixtures" tabindex="-1">Materialize API-backed fixtures <a class="header-anchor" href="#materialize-api-backed-fixtures" aria-label="Permalink to &quot;Materialize API-backed fixtures&quot;">​</a></h3><p>An input that only points at external data, such as a pull request URL, snapshot id, or pair of commit SHAs, is not self-contained. Fetch it once and commit the rendered fixture before you expand the suite.</p><ol><li>Save the diff, metadata, and labels under <code>fixtures/</code> at pinned revisions.</li><li>Seed those files with <code>workspaceFiles</code>, or read them from the fixture directory.</li><li>Assert decisions and output shape against the saved evidence.</li><li>Keep a small <code>smoke</code> subset for any remaining live pipeline checks.</li></ol><p><code>maxConcurrency</code> limits parallel datapoints. It does not limit model or API fan-out inside one datapoint. Materialized fixtures prevent a large suite from exhausting provider and GitHub rate limits. The <a href="./../skills/evals/SKILL.html">evals skill</a> has the full fixture workflow.</p><h2 id="keep-improvements-with-regression-evals" tabindex="-1">Keep improvements with regression evals <a class="header-anchor" href="#keep-improvements-with-regression-evals" aria-label="Permalink to &quot;Keep improvements with regression evals&quot;">​</a></h2><p>Every <a href="./hillclimbing.html">hillclimb</a> round that keeps a change must land an eval that would have failed before the change. If you can&#39;t express the improvement as a gate (a <code>calledTool</code> shift, a bounded <code>action.result</code> count, an output-shape regex), the improvement is unverified, and it&#39;ll regress silently.</p><p>The rule cuts the other way too: never weaken an existing gate to make a round pass. That&#39;s the freeze line moving, and it turns your regression suite into a list of checks that no longer protect anything.</p><h2 id="compare-variants-on-live-traffic" tabindex="-1">Compare variants on live traffic <a class="header-anchor" href="#compare-variants-on-live-traffic" aria-label="Permalink to &quot;Compare variants on live traffic&quot;">​</a></h2><p>Use <code>defineAB</code> to compare variant metrics on live sessions. It is not a test runner and has no <code>agent-sdk ab</code> command. Keep <code>defineEval</code> as the regression ratchet. Eval sessions do not enroll or change live metrics. See <a href="./ab.html">Live A/B metrics</a> for assignment, behavior, collection, and inspection.</p><h2 id="what-s-next" tabindex="-1">What&#39;s next <a class="header-anchor" href="#what-s-next" aria-label="Permalink to &quot;What&#39;s next&quot;">​</a></h2><p>Continue with these pages:</p><ul><li><a href="./ab.html">Live A/B metrics</a>: sticky variants and cumulative metrics on live sessions</li><li><a href="./hillclimbing.html">Hillclimbing</a>: the loop evals make trustworthy</li><li><a href="./building-with-agents.html">Building agents with agents</a>: have a coding agent write the first suite</li><li><a href="./guides/github.html">GitHub guide</a>: deterministic webhook fixtures with <code>github replay</code></li><li><a href="./reference/sessions.html">Sessions and streaming</a>: the events <code>t.events</code> contains</li></ul>`,77)])])}const g=i(n,[["render",l]]);export{c as __pageData,g as default};

package/dist/docs/assets/evals.md.BIUoVZ6X.lean.js

@@ -0,0 +1 @@
+import{_ as i,c as a,o as e,ag as t}from"./chunks/framework.CAZyNGu9.js";const c=JSON.parse('{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agent-sdk eval, and use them as regression checks.","frontmatter":{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agent-sdk eval, and use them as regression checks."},"headers":[],"relativePath":"evals.md","filePath":"evals.md"}'),n={name:"evals.md"};function l(h,s,p,d,o,r){return e(),a("div",null,[...s[0]||(s[0]=[t("",77)])])}const g=i(n,[["render",l]]);export{c as __pageData,g as default};

package/dist/docs/assets/{example-agents_approval-buddy.md.DFGBYLcc.js → example-agents_approval-buddy.md.BhEfleVx.js}

@@ -1,10 +1,10 @@
-import{_ as a,c as t,o as i,ag as s}from"./chunks/framework.CAZyNGu9.js";const u=JSON.parse('{"title":"Keep PR approval policy deterministic with Approval Buddy","description":"Separate code-owned eligibility from model-owned review, then connect GitHub, Slack, subagents, durable storage, and evals.","frontmatter":{"title":"Keep PR approval policy deterministic with Approval Buddy","description":"Separate code-owned eligibility from model-owned review, then connect GitHub, Slack, subagents, durable storage, and evals."},"headers":[],"relativePath":"example-agents/approval-buddy.md","filePath":"example-agents/approval-buddy.md"}'),o={name:"example-agents/approval-buddy.md"};function r(l,e,n,d,p,h){return i(),t("div",null,[...e[0]||(e[0]=[s(`<h1 id="keep-pr-approval-policy-deterministic-with-approval-buddy" tabindex="-1">Keep PR approval policy deterministic with Approval Buddy <a class="header-anchor" href="#keep-pr-approval-policy-deterministic-with-approval-buddy" aria-label="Permalink to &quot;Keep PR approval policy deterministic with Approval Buddy&quot;">​</a></h1><p>Approval Buddy approves eligible pull requests from a fixed roster and declines every other request. GitHub still blocks self-approval when the stamp identity authored the PR. Code decides eligibility. The model prepares evidence, runs two specialist reviews, and passes their findings to the approval tool without changing the policy decision.</p><p>Use this example when an agent can make a judgment inside a workflow, but authorization and the final side effect must stay in deterministic code.</p><p><a href="./../../examples/approval-buddy/">Browse the Approval Buddy source.</a></p><h2 id="keep-approval-policy-in-code" tabindex="-1">Keep approval policy in code <a class="header-anchor" href="#keep-approval-policy-in-code" aria-label="Permalink to &quot;Keep approval policy in code&quot;">​</a></h2><p>Approval Buddy draws three hard boundaries:</p><ul><li><code>prepare_review</code> and <code>approve_pr</code> re-read the live PR and apply the same eligibility rules.</li><li>Two subagents inspect prepared evidence, but their findings never grant or block approval.</li><li>Only <code>approve_pr</code> posts the GitHub review.</li></ul><p>A spoofed webhook, Slack message, or model claim can&#39;t add someone to the buddy roster. The mutating tool checks the source of truth immediately before it acts.</p><h2 id="follow-the-intended-stamp-flow" tabindex="-1">Follow the intended stamp flow <a class="header-anchor" href="#follow-the-intended-stamp-flow" aria-label="Permalink to &quot;Follow the intended stamp flow&quot;">​</a></h2><p>The root instructions ask the model to run this sequence for a qualifying PR:</p><ol><li>A non-draft <code>pull_request</code> event arrives with action <code>opened</code>, <code>reopened</code>, or <code>ready_for_review</code>.</li><li>The GitHub channel checks its repository allowlist and starts a session.</li><li><code>turn.started</code> posts a pending commit status.</li><li>The model calls <code>prepare_review</code>.</li><li>Host code fetches the live PR. It checks the author, open state, merged state, and draft state.</li><li>A qualifying PR gets <code>pr/MANIFEST.md</code>, <code>pr/meta.json</code>, and <code>pr/diff.patch</code> in the session workspace. Diffs above 2,000,000 characters are truncated and marked in metadata.</li><li>The model calls both review subagents through the built-in <code>task</code> tool.</li><li>It concatenates their contracted replies and calls <code>approve_pr</code>.</li><li><code>approve_pr</code> re-runs eligibility, posts an <code>APPROVE</code> review, and returns the outcome.</li><li>The channel posts a final commit status. A self-approval block also gets a short timeline comment because no approval review can appear.</li></ol><p>Ineligible PRs skip evidence and subagents. The model still calls <code>approve_pr</code> so the deterministic tool returns the formal decline reason.</p><p>Steps 4 through 9 are prompt-driven. The channel doesn&#39;t enforce tool order or prove both subagents ran, and <code>approve_pr</code> accepts missing findings. A failed turn clears the pending status with a green non-blocking result without approving the PR.</p><h2 id="map-the-framework-features" tabindex="-1">Map the framework features <a class="header-anchor" href="#map-the-framework-features" aria-label="Permalink to &quot;Map the framework features&quot;">​</a></h2><table tabindex="0"><thead><tr><th>Capability</th><th>Source</th><th>Role</th></tr></thead><tbody><tr><td>Root agent and policy prompt</td><td><a href="../../examples/approval-buddy/agent/agent.ts"><code>agent/agent.ts</code></a>, <a href="./../../examples/approval-buddy/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Configure the local agent and describe orchestration order.</td></tr><tr><td>GitHub channel</td><td><a href="../../examples/approval-buddy/agent/channels/github.ts"><code>agent/channels/github.ts</code></a></td><td>Filter wakes, lease GitHub access, and publish status events.</td></tr><tr><td>Slack channel</td><td><a href="../../examples/approval-buddy/agent/channels/slack.ts"><code>agent/channels/slack.ts</code></a></td><td>Accept approval-bot stamp and qualification requests.</td></tr><tr><td>Server tools</td><td><a href="./../../examples/approval-buddy/agent/tools/"><code>agent/tools/</code></a></td><td>Prepare evidence, approve, list buddies, and search GIFs.</td></tr><tr><td>Deterministic policy</td><td><a href="../../examples/approval-buddy/agent/lib/approve.ts"><code>agent/lib/approve.ts</code></a>, <a href="../../examples/approval-buddy/agent/lib/buddies.ts"><code>agent/lib/buddies.ts</code></a></td><td>Own the roster and live eligibility checks.</td></tr><tr><td>Review subagents</td><td><a href="./../../examples/approval-buddy/agent/subagents/"><code>agent/subagents/</code></a></td><td>Run deep audit and code-quality passes over the same evidence.</td></tr><tr><td>Storage</td><td><a href="../../examples/approval-buddy/agent/storage.ts"><code>agent/storage.ts</code></a></td><td>Persist sessions and events with <code>cursorHostedStorage</code> (Bugbot <code>agent_serve_*</code>).</td></tr><tr><td>Evals and unit tests</td><td><a href="./../../examples/approval-buddy/evals/"><code>evals/</code></a>, <a href="./../../examples/approval-buddy/agent/lib/"><code>agent/lib/</code></a></td><td>Protect routing, output contracts, policy, and GitHub behavior.</td></tr></tbody></table><p>There are no authored skills, MCP connections, schedules, reminders, hooks, A/B experiments, sandbox seeds, or tool approvals.</p><h2 id="prepare-credentials" tabindex="-1">Prepare credentials <a class="header-anchor" href="#prepare-credentials" aria-label="Permalink to &quot;Prepare credentials&quot;">​</a></h2><p>You need:</p><ul><li>Node 22.13 or newer.</li><li>An agent-runtime credential.</li><li>GitHub access to read PRs, post reviews, create commit statuses, and post the self-approval visibility comment.</li></ul><p>Optional GIF selection uses:</p><ul><li><code>GIPHY_API_KEY</code> or <code>APPROVAL_BUDDY_GIPHY_API_KEY</code>,</li><li><code>APPROVAL_BUDDY_STAMP_GIF</code>, or</li><li>severity-specific <code>APPROVAL_BUDDY_STAMP_GIF_&lt;LEVEL&gt;</code> variables.</li></ul><p>If you enable Giphy in a hosted copy, declare its secret and <code>api.giphy.com</code> egress.</p><h2 id="validate-without-approving-a-pr" tabindex="-1">Validate without approving a PR <a class="header-anchor" href="#validate-without-approving-a-pr" aria-label="Permalink to &quot;Validate without approving a PR&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> info</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p>List the deterministic roster:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> call</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> list_buddies</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+import{_ as a,c as t,o as s,ag as i}from"./chunks/framework.CAZyNGu9.js";const u=JSON.parse('{"title":"Keep PR approval policy deterministic with Approval Buddy","description":"Separate code-owned eligibility from model-owned review, then connect GitHub, Slack, subagents, durable storage, and evals.","frontmatter":{"title":"Keep PR approval policy deterministic with Approval Buddy","description":"Separate code-owned eligibility from model-owned review, then connect GitHub, Slack, subagents, durable storage, and evals."},"headers":[],"relativePath":"example-agents/approval-buddy.md","filePath":"example-agents/approval-buddy.md"}'),o={name:"example-agents/approval-buddy.md"};function r(l,e,n,d,p,h){return s(),t("div",null,[...e[0]||(e[0]=[i(`<h1 id="keep-pr-approval-policy-deterministic-with-approval-buddy" tabindex="-1">Keep PR approval policy deterministic with Approval Buddy <a class="header-anchor" href="#keep-pr-approval-policy-deterministic-with-approval-buddy" aria-label="Permalink to &quot;Keep PR approval policy deterministic with Approval Buddy&quot;">​</a></h1><p>Approval Buddy approves eligible pull requests from a fixed roster and declines every other request. GitHub still blocks self-approval when the stamp identity authored the PR. Code decides eligibility. The model prepares evidence, runs two specialist reviews, and passes their findings to the approval tool without changing the policy decision.</p><p>Use this example when an agent can make a judgment inside a workflow, but authorization and the final side effect must stay in deterministic code.</p><p><a href="./../../examples/approval-buddy/">Browse the Approval Buddy source.</a></p><h2 id="keep-approval-policy-in-code" tabindex="-1">Keep approval policy in code <a class="header-anchor" href="#keep-approval-policy-in-code" aria-label="Permalink to &quot;Keep approval policy in code&quot;">​</a></h2><p>Approval Buddy draws three hard boundaries:</p><ul><li><code>prepare_review</code> and <code>approve_pr</code> re-read the live PR and apply the same eligibility rules.</li><li>Two subagents inspect prepared evidence, but their findings never grant or block approval.</li><li>Only <code>approve_pr</code> posts the GitHub review.</li></ul><p>A spoofed webhook, Slack message, or model claim can&#39;t add someone to the buddy roster. The mutating tool checks the source of truth immediately before it acts.</p><h2 id="follow-the-intended-stamp-flow" tabindex="-1">Follow the intended stamp flow <a class="header-anchor" href="#follow-the-intended-stamp-flow" aria-label="Permalink to &quot;Follow the intended stamp flow&quot;">​</a></h2><p>The root instructions ask the model to run this sequence for a qualifying PR:</p><ol><li>A non-draft <code>pull_request</code> event arrives with action <code>opened</code>, <code>reopened</code>, or <code>ready_for_review</code>.</li><li>The GitHub channel checks its repository allowlist and starts a session.</li><li><code>turn.started</code> posts a pending commit status.</li><li>The model calls <code>prepare_review</code>.</li><li>Host code fetches the live PR. It checks the author, open state, merged state, and draft state.</li><li>A qualifying PR gets <code>pr/MANIFEST.md</code>, <code>pr/meta.json</code>, and <code>pr/diff.patch</code> in the session workspace. Diffs above 2,000,000 characters are truncated and marked in metadata.</li><li>The model calls both review subagents through the built-in <code>task</code> tool.</li><li>It concatenates their contracted replies and calls <code>approve_pr</code>.</li><li><code>approve_pr</code> re-runs eligibility, posts an <code>APPROVE</code> review, and returns the outcome.</li><li>The channel posts a final commit status. A self-approval block also gets a short timeline comment because no approval review can appear.</li></ol><p>Ineligible PRs skip evidence and subagents. The model still calls <code>approve_pr</code> so the deterministic tool returns the formal decline reason.</p><p>Steps 4 through 9 are prompt-driven. The channel doesn&#39;t enforce tool order or prove both subagents ran, and <code>approve_pr</code> accepts missing findings. A failed turn clears the pending status with a green non-blocking result without approving the PR.</p><h2 id="map-the-framework-features" tabindex="-1">Map the framework features <a class="header-anchor" href="#map-the-framework-features" aria-label="Permalink to &quot;Map the framework features&quot;">​</a></h2><table tabindex="0"><thead><tr><th>Capability</th><th>Source</th><th>Role</th></tr></thead><tbody><tr><td>Root agent and policy prompt</td><td><a href="../../examples/approval-buddy/agent/agent.ts"><code>agent/agent.ts</code></a>, <a href="./../../examples/approval-buddy/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Configure the local agent and describe orchestration order.</td></tr><tr><td>GitHub channel</td><td><a href="../../examples/approval-buddy/agent/channels/github.ts"><code>agent/channels/github.ts</code></a></td><td>Filter wakes, lease GitHub access, and publish status events.</td></tr><tr><td>Slack channel</td><td><a href="../../examples/approval-buddy/agent/channels/slack.ts"><code>agent/channels/slack.ts</code></a></td><td>Accept approval-bot stamp and qualification requests.</td></tr><tr><td>Server tools</td><td><a href="./../../examples/approval-buddy/agent/tools/"><code>agent/tools/</code></a></td><td>Prepare evidence, approve, list buddies, and search GIFs.</td></tr><tr><td>Deterministic policy</td><td><a href="../../examples/approval-buddy/agent/lib/approve.ts"><code>agent/lib/approve.ts</code></a>, <a href="../../examples/approval-buddy/agent/lib/buddies.ts"><code>agent/lib/buddies.ts</code></a></td><td>Own the roster and live eligibility checks.</td></tr><tr><td>Review subagents</td><td><a href="./../../examples/approval-buddy/agent/subagents/"><code>agent/subagents/</code></a></td><td>Run deep audit and code-quality passes over the same evidence.</td></tr><tr><td>Storage</td><td><a href="../../examples/approval-buddy/agent/storage.ts"><code>agent/storage.ts</code></a></td><td>Persist sessions and events with <code>cursorHostedStorage</code> (Bugbot <code>agent_serve_*</code>).</td></tr><tr><td>Evals and unit tests</td><td><a href="./../../examples/approval-buddy/evals/"><code>evals/</code></a>, <a href="./../../examples/approval-buddy/agent/lib/"><code>agent/lib/</code></a></td><td>Protect routing, output contracts, policy, and GitHub behavior.</td></tr></tbody></table><p>There are no authored skills, MCP connections, schedules, reminders, hooks, A/B experiments, sandbox seeds, or tool approvals.</p><h2 id="prepare-credentials" tabindex="-1">Prepare credentials <a class="header-anchor" href="#prepare-credentials" aria-label="Permalink to &quot;Prepare credentials&quot;">​</a></h2><p>You need:</p><ul><li>Node 22.13 or newer.</li><li>An agent-runtime credential.</li><li>GitHub access to read PRs, post reviews, create commit statuses, and post the self-approval visibility comment.</li></ul><p>Optional GIF selection uses:</p><ul><li><code>GIPHY_API_KEY</code> or <code>APPROVAL_BUDDY_GIPHY_API_KEY</code>,</li><li><code>APPROVAL_BUDDY_STAMP_GIF</code>, or</li><li>severity-specific <code>APPROVAL_BUDDY_STAMP_GIF_&lt;LEVEL&gt;</code> variables.</li></ul><p>If you enable Giphy in a hosted copy, declare its secret and <code>api.giphy.com</code> egress.</p><h2 id="validate-without-approving-a-pr" tabindex="-1">Validate without approving a PR <a class="header-anchor" href="#validate-without-approving-a-pr" aria-label="Permalink to &quot;Validate without approving a PR&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> info</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p>List the deterministic roster:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> call</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> list_buddies</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --input</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &#39;{}&#39;</span></span></code></pre></div><p>Set a known merged PR, then run the read-only precheck:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">MERGED_PR_URL</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">https://github.com/your-org/your-repo/pull/123</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> call</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> prepare_review</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> call</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> prepare_review</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
-<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --input</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;{</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\\&quot;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">prUrl</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\\&quot;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">:</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\\&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$MERGED_PR_URL</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\\&quot;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">}&quot;</span></span></code></pre></div><p>The result should decline because the PR is no longer open. <code>prepare_review</code> never posts an approval.</p><div class="caution custom-block github-alert"><p class="custom-block-title">CAUTION</p><p>Don&#39;t use <code>agentkit call approve_pr</code> as a smoke test. The tool has no <code>needsApproval</code> gate and posts a real GitHub review when the PR qualifies.</p></div><h2 id="see-why-preparation-is-separate" tabindex="-1">See why preparation is separate <a class="header-anchor" href="#see-why-preparation-is-separate" aria-label="Permalink to &quot;See why preparation is separate&quot;">​</a></h2><p><code>prepare_review</code> is read-only. It checks policy before fetching a large diff, so declined requests don&#39;t spend review-agent work.</p><p>Direct calls return the evidence file map because their scratch workspace is deleted after the call. In-session calls write the tree to <code>ctx.workspaceDir</code>, where both subagents can read it.</p><p><code>approve_pr</code> repeats the live check instead of trusting preparation. A PR can close, merge, become a draft, or change author-related context between the two steps. Revalidation keeps the final write bound to current state.</p><p>This is a reusable two-tool pattern:</p><ul><li>a read-only tool prepares and explains the decision,</li><li>a mutating tool repeats policy at the side-effect boundary.</li></ul><h2 id="fan-out-two-review-contracts" tabindex="-1">Fan out two review contracts <a class="header-anchor" href="#fan-out-two-review-contracts" aria-label="Permalink to &quot;Fan out two review contracts&quot;">​</a></h2><p>The two discovered subagents have different contracts:</p><ul><li>The security reviewer reports bugs, breaking changes, and security findings with <code>High</code>, <code>Medium</code>, or <code>Low</code> tags.</li><li>The code-quality reviewer reports maintainability and structure concerns with <code>Blocker</code>, <code>Major</code>, or <code>Minor</code> tags.</li></ul><p>The parent calls both through the harness <code>task</code> tool. They inherit the root agent&#39;s execution surface and read the same <code>pr/</code> workspace. The prompt asks the parent not to rewrite either reply. The review body trims the combined text and caps it at 16,000 characters.</p><p>Findings are informational. A high-severity finding doesn&#39;t veto the stamp. That policy is explicit in the root instructions and approval code.</p><h2 id="trace-github-channel-behavior" tabindex="-1">Trace GitHub channel behavior <a class="header-anchor" href="#trace-github-channel-behavior" aria-label="Permalink to &quot;Trace GitHub channel behavior&quot;">​</a></h2><p>The channel uses <code>githubChannel</code> with:</p><ul><li>a configured repository allowlist on the account-linked GitHub transport,</li><li>a second optional <code>APPROVAL_BUDDY_REPOS</code> wake filter,</li><li><code>deliverReplies: false</code>,</li><li>progress reactions disabled, and</li><li>event handlers for turn start, <code>approve_pr</code> results, and failed turns.</li></ul><p>The source requests <code>contents-write</code>, even though the documented workflow posts reviews, statuses, and comments. When adapting the example, start with <code>pr-write</code> and opt up only if a tool must push code.</p><p>Every terminal status is green by design. Declines and crashed turns are informational, not merge-blocking. This is a product decision in the example, not an agentkit default.</p><p>A successful turn that never calls <code>approve_pr</code> leaves the pending status in place. The channel clears it on <code>approve_pr</code> results and <code>turn.failed</code>, but has no <code>turn.completed</code> fallback.</p><p><code>github replay</code> reaches the same channel and can post a real approval, status, or comment. Use replay only against a repository and PR created for this test.</p><h2 id="use-slack-for-explicit-requests" tabindex="-1">Use Slack for explicit requests <a class="header-anchor" href="#use-slack-for-explicit-requests" aria-label="Permalink to &quot;Use Slack for explicit requests&quot;">​</a></h2><p>Start the dev server:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span></span></code></pre></div><p>Then ask through the signed-in account-linked Slack connection:</p><blockquote><p>Would this PR qualify for a stamp?</p></blockquote><p>The instructions route qualification questions to <code>prepare_review</code> only. A stamp request runs the complete flow and may approve the PR.</p><p>This channel uses the account-linked transport instead of a dedicated Socket Mode app.</p><h2 id="see-how-durable-storage-fits" tabindex="-1">See how durable storage fits <a class="header-anchor" href="#see-how-durable-storage-fits" aria-label="Permalink to &quot;See how durable storage fits&quot;">​</a></h2><p><code>defineStorage</code> replaces the default local session store with a shared, durable key-value adapter. Approval Buddy chooses:</p><ul><li>a 15-second write debounce,</li><li>startup restoration for up to 200 sessions, and</li><li>a 14-day restore window.</li></ul><p>That policy fits long-lived Slack threads and a small webhook fleet. The security reviewer uses the same adapter with lazy restore, which fits its shorter sessions.</p><h2 id="run-the-regression-suite" tabindex="-1">Run the regression suite <a class="header-anchor" href="#run-the-regression-suite" aria-label="Permalink to &quot;Run the regression suite&quot;">​</a></h2><p>List the four eval cases:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span></span></code></pre></div><p>The suite covers:</p><ul><li>buddy-list routing,</li><li>declining a merged PR,</li><li>using only <code>prepare_review</code> for a qualification question, and</li><li>the combined findings headings and severity format over seeded evidence.</li></ul><p>Run the safe qualification case:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --input</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;{</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\\&quot;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">prUrl</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\\&quot;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">:</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\\&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$MERGED_PR_URL</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\\&quot;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">}&quot;</span></span></code></pre></div><p>The result should decline because the PR is no longer open. <code>prepare_review</code> never posts an approval.</p><div class="caution custom-block github-alert"><p class="custom-block-title">CAUTION</p><p>Don&#39;t use <code>agent-sdk call approve_pr</code> as a smoke test. The tool has no <code>needsApproval</code> gate and posts a real GitHub review when the PR qualifies.</p></div><h2 id="see-why-preparation-is-separate" tabindex="-1">See why preparation is separate <a class="header-anchor" href="#see-why-preparation-is-separate" aria-label="Permalink to &quot;See why preparation is separate&quot;">​</a></h2><p><code>prepare_review</code> is read-only. It checks policy before fetching a large diff, so declined requests don&#39;t spend review-agent work.</p><p>Direct calls return the evidence file map because their scratch workspace is deleted after the call. In-session calls write the tree to <code>ctx.workspaceDir</code>, where both subagents can read it.</p><p><code>approve_pr</code> repeats the live check instead of trusting preparation. A PR can close, merge, become a draft, or change author-related context between the two steps. Revalidation keeps the final write bound to current state.</p><p>This is a reusable two-tool pattern:</p><ul><li>a read-only tool prepares and explains the decision,</li><li>a mutating tool repeats policy at the side-effect boundary.</li></ul><h2 id="fan-out-two-review-contracts" tabindex="-1">Fan out two review contracts <a class="header-anchor" href="#fan-out-two-review-contracts" aria-label="Permalink to &quot;Fan out two review contracts&quot;">​</a></h2><p>The two discovered subagents have different contracts:</p><ul><li>The security reviewer reports bugs, breaking changes, and security findings with <code>High</code>, <code>Medium</code>, or <code>Low</code> tags.</li><li>The code-quality reviewer reports maintainability and structure concerns with <code>Blocker</code>, <code>Major</code>, or <code>Minor</code> tags.</li></ul><p>The parent calls both through the harness <code>task</code> tool. They inherit the root agent&#39;s execution surface and read the same <code>pr/</code> workspace. The prompt asks the parent not to rewrite either reply. The review body trims the combined text and caps it at 16,000 characters.</p><p>Findings are informational. A high-severity finding doesn&#39;t veto the stamp. That policy is explicit in the root instructions and approval code.</p><h2 id="trace-github-channel-behavior" tabindex="-1">Trace GitHub channel behavior <a class="header-anchor" href="#trace-github-channel-behavior" aria-label="Permalink to &quot;Trace GitHub channel behavior&quot;">​</a></h2><p>The channel uses <code>githubChannel</code> with:</p><ul><li>a configured repository allowlist on the account-linked GitHub transport,</li><li>a second optional <code>APPROVAL_BUDDY_REPOS</code> wake filter,</li><li><code>deliverReplies: false</code>,</li><li>progress reactions disabled, and</li><li>event handlers for turn start, <code>approve_pr</code> results, and failed turns.</li></ul><p>The source requests <code>contents-write</code>, even though the documented workflow posts reviews, statuses, and comments. When adapting the example, start with <code>pr-write</code> and opt up only if a tool must push code.</p><p>Every terminal status is green by design. Declines and crashed turns are informational, not merge-blocking. This is a product decision in the example, not an Agent SDK default.</p><p>A successful turn that never calls <code>approve_pr</code> leaves the pending status in place. The channel clears it on <code>approve_pr</code> results and <code>turn.failed</code>, but has no <code>turn.completed</code> fallback.</p><p><code>github replay</code> reaches the same channel and can post a real approval, status, or comment. Use replay only against a repository and PR created for this test.</p><h2 id="use-slack-for-explicit-requests" tabindex="-1">Use Slack for explicit requests <a class="header-anchor" href="#use-slack-for-explicit-requests" aria-label="Permalink to &quot;Use Slack for explicit requests&quot;">​</a></h2><p>Start the dev server:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span></span></code></pre></div><p>Then ask through the signed-in account-linked Slack connection:</p><blockquote><p>Would this PR qualify for a stamp?</p></blockquote><p>The instructions route qualification questions to <code>prepare_review</code> only. A stamp request runs the complete flow and may approve the PR.</p><p>This channel uses the account-linked transport instead of a dedicated Socket Mode app.</p><h2 id="see-how-durable-storage-fits" tabindex="-1">See how durable storage fits <a class="header-anchor" href="#see-how-durable-storage-fits" aria-label="Permalink to &quot;See how durable storage fits&quot;">​</a></h2><p><code>defineStorage</code> replaces the default local session store with a shared, durable key-value adapter. Approval Buddy chooses:</p><ul><li>a 15-second write debounce,</li><li>startup restoration for up to 200 sessions, and</li><li>a 14-day restore window.</li></ul><p>That policy fits long-lived Slack threads and a small webhook fleet. The security reviewer uses the same adapter with lazy restore, which fits its shorter sessions.</p><h2 id="run-the-regression-suite" tabindex="-1">Run the regression suite <a class="header-anchor" href="#run-the-regression-suite" aria-label="Permalink to &quot;Run the regression suite&quot;">​</a></h2><p>List the four eval cases:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span></span></code></pre></div><p>The suite covers:</p><ul><li>buddy-list routing,</li><li>declining a merged PR,</li><li>using only <code>prepare_review</code> for a qualification question, and</li><li>the combined findings headings and severity format over seeded evidence.</li></ul><p>Run the safe qualification case:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">  qualify/merged-pr-question</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --json</span></span></code></pre></div><p>The qualification case reads a live merged PR. The seeded format case shown by <code>--list</code> uses a planted auth-bypass diff and checks for a <code>task</code> call, both headings, and severity tags. It doesn&#39;t prove both named subagents ran or whether their output reached <code>approve_pr</code>. Unit tests under <code>agent/lib/</code> cover policy, self-approval handling, evidence limits, status mapping, GIF selection, and severity parsing.</p><h2 id="reuse-the-policy-boundary" tabindex="-1">Reuse the policy boundary <a class="header-anchor" href="#reuse-the-policy-boundary" aria-label="Permalink to &quot;Reuse the policy boundary&quot;">​</a></h2><p>Keep these properties when you replace the buddy policy:</p><ol><li>Put authorization in typed code.</li><li>Fetch the source of truth inside both prepare and mutate steps.</li><li>Give the model evidence only after the request qualifies.</li><li>Treat specialist findings as data, not authority.</li><li>Keep the core domain mutation in one named tool. Treat channel status and visibility writes as separate, audited effects.</li><li>Add a human approval gate if your policy still needs operator consent.</li><li>Test read-only routing separately from mutation.</li></ol><h2 id="where-to-go-next" tabindex="-1">Where to go next <a class="header-anchor" href="#where-to-go-next" aria-label="Permalink to &quot;Where to go next&quot;">​</a></h2><ul><li><a href="./../guides/github.html">GitHub</a></li><li><a href="./../reference/tools.html">Tools</a></li><li><a href="./../reference/subagents.html">Subagents</a></li><li><a href="./../storage.html">Storage</a></li><li><a href="./../guides/slack.html">Slack</a></li><li><a href="./../evals.html">Evals</a></li></ul>`,72)])])}const k=a(o,[["render",r]]);export{u as __pageData,k as default};

package/dist/docs/assets/{example-agents_approval-buddy.md.DFGBYLcc.lean.js → example-agents_approval-buddy.md.BhEfleVx.lean.js}

@@ -1 +1 @@
-import{_ as a,c as t,o as i,ag as s}from"./chunks/framework.CAZyNGu9.js";const u=JSON.parse('{"title":"Keep PR approval policy deterministic with Approval Buddy","description":"Separate code-owned eligibility from model-owned review, then connect GitHub, Slack, subagents, durable storage, and evals.","frontmatter":{"title":"Keep PR approval policy deterministic with Approval Buddy","description":"Separate code-owned eligibility from model-owned review, then connect GitHub, Slack, subagents, durable storage, and evals."},"headers":[],"relativePath":"example-agents/approval-buddy.md","filePath":"example-agents/approval-buddy.md"}'),o={name:"example-agents/approval-buddy.md"};function r(l,e,n,d,p,h){return i(),t("div",null,[...e[0]||(e[0]=[s("",72)])])}const k=a(o,[["render",r]]);export{u as __pageData,k as default};
+import{_ as a,c as t,o as s,ag as i}from"./chunks/framework.CAZyNGu9.js";const u=JSON.parse('{"title":"Keep PR approval policy deterministic with Approval Buddy","description":"Separate code-owned eligibility from model-owned review, then connect GitHub, Slack, subagents, durable storage, and evals.","frontmatter":{"title":"Keep PR approval policy deterministic with Approval Buddy","description":"Separate code-owned eligibility from model-owned review, then connect GitHub, Slack, subagents, durable storage, and evals."},"headers":[],"relativePath":"example-agents/approval-buddy.md","filePath":"example-agents/approval-buddy.md"}'),o={name:"example-agents/approval-buddy.md"};function r(l,e,n,d,p,h){return s(),t("div",null,[...e[0]||(e[0]=[i("",72)])])}const k=a(o,[["render",r]]);export{u as __pageData,k as default};

package/dist/docs/assets/example-agents_benny.md.2Et1qa8f.js

@@ -0,0 +1,7 @@
+import{_ as a,c as t,o as s,ag as i}from"./chunks/framework.CAZyNGu9.js";const k=JSON.parse('{"title":"Route Slack work through repository playbooks","description":"Combine account-linked chat, allowlisted Socket Mode channel watching, inherited repository skills, and a custom local workspace.","frontmatter":{"title":"Route Slack work through repository playbooks","description":"Combine account-linked chat, allowlisted Socket Mode channel watching, inherited repository skills, and a custom local workspace."},"headers":[],"relativePath":"example-agents/benny.md","filePath":"example-agents/benny.md"}'),o={name:"example-agents/benny.md"};function n(l,e,r,h,d,p){return s(),t("div",null,[...e[0]||(e[0]=[i(`<h1 id="route-slack-work-through-repository-playbooks" tabindex="-1">Route Slack work through repository playbooks <a class="header-anchor" href="#route-slack-work-through-repository-playbooks" aria-label="Permalink to &quot;Route Slack work through repository playbooks&quot;">​</a></h1><p>This agent is a Slack teammate for a product team. Mentions and direct messages reach it through an account-linked transport. New top-level posts in an allowlisted issue channel reach it through a dedicated Slack app, even without a mention. The agent then selects a repository playbook for triage, reproduction, fixes, reviews, on-call work, or design critique.</p><p>Use this example when Slack is the intake surface and your durable procedures already live as repository skills.</p><p><a href="./../../examples/benny/">Browse the current playbook-router source.</a></p><h2 id="combine-two-slack-transports-with-repo-skills" tabindex="-1">Combine two Slack transports with repo skills <a class="header-anchor" href="#combine-two-slack-transports-with-repo-skills" aria-label="Permalink to &quot;Combine two Slack transports with repo skills&quot;">​</a></h2><p>The playbook router uniquely combines three decisions:</p><ul><li>Two Slack transports serve different engagement modes.</li><li><code>local.cwd</code> keeps session workspaces inside the monorepo.</li><li>Instructions route work to inherited repository playbooks instead of authored <code>agent/skills/</code>.</li></ul><p>The result is a thin agent project over a mature procedure library.</p><h2 id="follow-an-issue-report" tabindex="-1">Follow an issue report <a class="header-anchor" href="#follow-an-issue-report" aria-label="Permalink to &quot;Follow an issue report&quot;">​</a></h2><ol><li>A teammate creates a top-level post in the allowlisted issue channel.</li><li>The dedicated Socket Mode channel accepts the allowlisted channel.</li><li>A 15-second debounce lets edits settle. Deleting the post during that window cancels the dispatch.</li><li>The Agent SDK creates a thread-scoped session and sends the report to the playbook router.</li><li>The instructions select the matching triage playbook.</li><li>The harness finds the repository root, opens the inherited playbook, and follows its procedure.</li><li>The agent posts only in the source thread and reports the evidence it gathered.</li></ol><p>Mentions and direct messages follow the same agent instructions. They don&#39;t need the watched-channel path.</p><h2 id="map-the-playbook-router-files" tabindex="-1">Map the playbook router files <a class="header-anchor" href="#map-the-playbook-router-files" aria-label="Permalink to &quot;Map the playbook router files&quot;">​</a></h2><table tabindex="0"><thead><tr><th>File</th><th>Purpose</th></tr></thead><tbody><tr><td><a href="../../examples/benny/agent/agent.ts"><code>agent/agent.ts</code></a></td><td>Names the agent, selects its model, and keeps the harness under <code>.agent-serve/harness</code>.</td></tr><tr><td><a href="./../../examples/benny/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Defines engagement rules, evidence policy, and the playbook routing map.</td></tr><tr><td><a href="../../examples/benny/agent/channels/slack.ts"><code>agent/channels/slack.ts</code></a></td><td>Handles account-linked mentions and direct messages.</td></tr><tr><td><a href="../../examples/benny/agent/channels/slack-app.ts"><code>agent/channels/slack-app.ts</code></a></td><td>Runs the dedicated app and watches one allowlisted channel.</td></tr><tr><td><a href="../../examples/benny/evals/smoke.eval.ts"><code>evals/smoke.eval.ts</code></a></td><td>Checks the agent identity and expected triage route.</td></tr></tbody></table><p>The playbook router authors no tools, MCP connections, subagents, schedules, hooks, A/B experiments, or sandbox seeds.</p><h2 id="see-why-local-cwd-matters" tabindex="-1">See why <code>local.cwd</code> matters <a class="header-anchor" href="#see-why-local-cwd-matters" aria-label="Permalink to &quot;See why \`local.cwd\` matters&quot;">​</a></h2><p>The Agent SDK normally keeps an ephemeral <code>run</code> or <code>eval</code> workspace outside a large monorepo. This prevents ancestor instruction and repository-rule files from leaking into an unrelated agent.</p><p>The playbook router needs the opposite. Its procedures live at the repository root, so <code>agent.ts</code> sets:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">local</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">  cwd</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;.agent-serve/harness&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Each harness workspace lands under <code>examples/benny/.agent-serve/harness/&lt;sessionId&gt;</code>. Walking up the directory tree reaches the host repository and its inherited playbook directory.</p><p>Those playbooks are inherited context. <code>agent-sdk info</code> reports zero authored skills for the agent. Copying this project into another repository removes its main procedures unless you copy or replace the skill library too.</p><h2 id="connect-both-slack-paths" tabindex="-1">Connect both Slack paths <a class="header-anchor" href="#connect-both-slack-paths" aria-label="Permalink to &quot;Connect both Slack paths&quot;">​</a></h2><p>The account-linked path needs an agent-runtime login and a connected Slack account:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> login</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> whoami</span></span></code></pre></div><p>It routes explicit mentions without a dedicated Slack token on the host.</p><p>For the watched-channel path, configure a dedicated Socket Mode app with:</p><ul><li>subscribe to <code>message.channels</code> and <code>message.groups</code>,</li><li>have an App-Level Token with <code>connections:write</code>, and</li><li>be a member of the watched channel.</li></ul><p>Run <code>agent-sdk slack setup</code> for the guided app workflow. Generate the project manifest with <code>--channel-posts</code> when you create a new copy, then validate the configured channel prefix with <code>agent-sdk slack doctor</code>.</p><p>Missing dedicated-app tokens leave that channel idle. They don&#39;t stop the account-linked channel.</p><h2 id="validate-and-start-the-server" tabindex="-1">Validate and start the server <a class="header-anchor" href="#validate-and-start-the-server" aria-label="Permalink to &quot;Validate and start the server&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/benny</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> info</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/benny</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/benny</span></span></code></pre></div><p>The info output should show two Slack channels and no authored skill. That combination confirms the example is using inherited playbooks.</p><h2 id="exercise-each-engagement-mode" tabindex="-1">Exercise each engagement mode <a class="header-anchor" href="#exercise-each-engagement-mode" aria-label="Permalink to &quot;Exercise each engagement mode&quot;">​</a></h2><p>Test the explicit account-linked path by asking:</p><blockquote><p>Which playbook would you use to triage a product UI bug?</p></blockquote><p>Test the dedicated app:</p><ol><li>Create a top-level post in the allowlisted issue channel.</li><li>Don&#39;t mention the bot.</li><li>Wait for the debounce window.</li><li>Confirm the agent replies in the post&#39;s thread.</li></ol><p>Thread replies don&#39;t trigger the proactive watch. Mentions still use Slack&#39;s normal mention path. Bot-authored posts are ignored to prevent loops.</p><p>The channel uses the default handler after filtering. It doesn&#39;t apply a second code-level classifier, so every accepted top-level post spends a model turn and reaches the prompt.</p><h2 id="inspect-thread-continuity" tabindex="-1">Inspect thread continuity <a class="header-anchor" href="#inspect-thread-continuity" aria-label="Permalink to &quot;Inspect thread continuity&quot;">​</a></h2><p>The Agent SDK keys Slack sessions by channel and thread timestamp. A follow-up in the same thread resumes the conversation and workspace. A new top-level issue gets a new session.</p><p>This lets a playbook gather evidence over several turns without mixing two reports. The playground shows both the account-linked and dedicated-app sessions while the dev server runs.</p><h2 id="run-the-smoke-eval" tabindex="-1">Run the smoke eval <a class="header-anchor" href="#run-the-smoke-eval" aria-label="Permalink to &quot;Run the smoke eval&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/benny</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/benny</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> smoke</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p>The case asks for the agent identity and the playbook used for issue triage. It checks the configured identity and route label.</p><p>This is a lexical smoke test. It doesn&#39;t prove Slack delivery, skill selection, skill loading, procedure execution, or thread-only behavior. Add fixture-backed evals around the playbooks when you reuse this design.</p><h2 id="build-a-playbook-routed-teammate" tabindex="-1">Build a playbook-routed teammate <a class="header-anchor" href="#build-a-playbook-routed-teammate" aria-label="Permalink to &quot;Build a playbook-routed teammate&quot;">​</a></h2><p>Use this structure when your organization already has tested skills:</p><ol><li>Put the playbooks under a stable repository path.</li><li>Set <code>local.cwd</code> so harness workspaces can inherit that path.</li><li>Write a short routing table in <code>instructions.md</code>.</li><li>Use account-linked Slack for explicit requests.</li><li>Add a dedicated app only for allowlisted proactive intake.</li><li>Keep the channel allowlist narrow and debounce edited posts.</li><li>Add an eval for every important request-to-playbook route.</li></ol><p>If the procedures should ship with the agent, put them under <code>agent/skills/</code> instead. Authored skills appear in the manifest and travel with the project.</p><h2 id="where-to-go-next" tabindex="-1">Where to go next <a class="header-anchor" href="#where-to-go-next" aria-label="Permalink to &quot;Where to go next&quot;">​</a></h2><ul><li><a href="./../guides/slack.html">Slack</a></li><li><a href="./../reference/agent-config.html">Agent config</a></li><li><a href="./../reference/skills.html">Skills</a></li><li><a href="./../reference/sessions.html">Sessions and streaming</a></li><li><a href="./../evals.html">Evals</a></li></ul>`,51)])])}const u=a(o,[["render",n]]);export{k as __pageData,u as default};

package/dist/docs/assets/{example-agents_bugbot.md.DelIdhxB.js → example-agents_bugbot.md.ByUexi5i.js}

@@ -1,11 +1,11 @@
-import{_ as a,c as t,o as s,ag as i}from"./chunks/framework.CAZyNGu9.js";const u=JSON.parse('{"title":"Review prepared pull-request evidence","description":"Fetch a PR on the host, seed a trimmed diff-first workspace, and run a GitHub-read-only review through HTTP, Slack, or an eval.","frontmatter":{"title":"Review prepared pull-request evidence","description":"Fetch a PR on the host, seed a trimmed diff-first workspace, and run a GitHub-read-only review through HTTP, Slack, or an eval."},"headers":[],"relativePath":"example-agents/bugbot.md","filePath":"example-agents/bugbot.md"}'),r={name:"example-agents/bugbot.md"};function n(l,e,o,h,d,p){return s(),t("div",null,[...e[0]||(e[0]=[i(`<h1 id="review-prepared-pull-request-evidence" tabindex="-1">Review prepared pull-request evidence <a class="header-anchor" href="#review-prepared-pull-request-evidence" aria-label="Permalink to &quot;Review prepared pull-request evidence&quot;">​</a></h1><p>This GitHub-read-only reviewer uses host code to fetch the PR with <code>gh</code> and <code>git</code>, builds a trimmed <code>pr/</code> evidence tree, then hands that tree to the model. The model reads the diff, loads a review skill, and returns at most three high-confidence findings.</p><p>Use this example when the host should control evidence collection and the model shouldn&#39;t browse or mutate the source repository.</p><p><a href="./../../examples/bugbot/">Browse the current reviewer source.</a></p><h2 id="separate-evidence-preparation-from-review" tabindex="-1">Separate evidence preparation from review <a class="header-anchor" href="#separate-evidence-preparation-from-review" aria-label="Permalink to &quot;Separate evidence preparation from review&quot;">​</a></h2><p>The reviewer separates preparation from judgment:</p><ul><li>Host code owns GitHub and Git access.</li><li>A server tool turns untrusted PR input into bounded workspace files.</li><li>A custom channel seeds those files before the model starts.</li><li>An on-demand skill defines the review procedure and output contract.</li><li>The model returns chat text. No path posts a GitHub review.</li></ul><p>This architecture gives the model a purpose-built evidence package instead of a checkout.</p><h2 id="follow-a-review" tabindex="-1">Follow a review <a class="header-anchor" href="#follow-a-review" aria-label="Permalink to &quot;Follow a review&quot;">​</a></h2><p>The custom HTTP path runs this sequence:</p><ol><li><code>POST /v1/channels/review/</code> receives a PR reference.</li><li>The handler calls <code>prepare_pr</code> without a model turn.</li><li>Host code reads PR metadata and the unified diff.</li><li>It reuses a matching checkout, force-fetching the PR ref there when the commit is missing. Without a matching checkout, it uses a temporary bare cache.</li><li>It creates <code>pr/MANIFEST.md</code>, <code>pr/meta.json</code>, <code>pr/diff.patch</code>, and selected small files and rules.</li><li><code>send({ workspaceFiles })</code> creates the model session with that evidence.</li><li>The model reads the manifest and diff, then loads <code>pr-review</code>.</li><li>The channel returns session and playground URLs while the review streams.</li></ol><p>If a normal chat starts without evidence, the model can call <code>prepare_pr</code> mid-turn. That form writes the same files into the active session workspace.</p><h2 id="map-the-evidence-review-files" tabindex="-1">Map the evidence-review files <a class="header-anchor" href="#map-the-evidence-review-files" aria-label="Permalink to &quot;Map the evidence-review files&quot;">​</a></h2><table tabindex="0"><thead><tr><th>File</th><th>Purpose</th></tr></thead><tbody><tr><td><a href="../../examples/bugbot/agent/agent.ts"><code>agent/agent.ts</code></a></td><td>Selects the local runtime and model.</td></tr><tr><td><a href="./../../examples/bugbot/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Requires diff-first review and confines model work to <code>pr/</code>.</td></tr><tr><td><a href="../../examples/bugbot/agent/tools/prepare_pr.ts"><code>agent/tools/prepare_pr.ts</code></a></td><td>Exposes host preparation as a typed server tool.</td></tr><tr><td><a href="../../examples/bugbot/agent/lib/prepare-pr.ts"><code>agent/lib/prepare-pr.ts</code></a></td><td>Parses PR references, runs <code>gh</code> and <code>git</code>, and builds the evidence map.</td></tr><tr><td><a href="../../examples/bugbot/agent/channels/review.ts"><code>agent/channels/review.ts</code></a></td><td>Provides the loopback-only prepare-and-send HTTP route.</td></tr><tr><td><a href="../../examples/bugbot/agent/channels/slack.ts"><code>agent/channels/slack.ts</code></a></td><td>Extracts PR references and prepares evidence for mentions and direct messages.</td></tr><tr><td><a href="./../../examples/bugbot/agent/skills/pr-review.html"><code>agent/skills/pr-review.md</code></a></td><td>Sets finding limits, severities, and the machine-readable review format.</td></tr><tr><td><a href="../../examples/bugbot/evals/review/smoke.eval.ts"><code>evals/review/smoke.eval.ts</code></a></td><td>Seeds fake evidence and checks the review path without GitHub.</td></tr></tbody></table><p>There is no authored GitHub channel, MCP connection, subagent, schedule, hook, A/B experiment, approval, or custom storage.</p><h2 id="prepare-the-host" tabindex="-1">Prepare the host <a class="header-anchor" href="#prepare-the-host" aria-label="Permalink to &quot;Prepare the host&quot;">​</a></h2><p>You need:</p><ul><li>Node 22.13 or newer.</li><li>An agent-runtime credential for model turns and account-linked Slack.</li><li><code>gh</code> and <code>git</code> on <code>PATH</code>.</li><li><code>gh</code> access to the target PR.</li><li>Network access to GitHub and a writable temporary directory.</li></ul><p>The preparer can prefer a configured local checkout. Its <code>origin</code> must match the target repository. Otherwise the reviewer uses its bare cache. It never checks out the PR into the serve host&#39;s working tree.</p><h2 id="validate-the-surface" tabindex="-1">Validate the surface <a class="header-anchor" href="#validate-the-surface" aria-label="Permalink to &quot;Validate the surface&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/bugbot</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> info</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/bugbot</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p>The manifest should show one server tool, one skill, and two authored channels.</p><h2 id="inspect-evidence-without-a-model-turn" tabindex="-1">Inspect evidence without a model turn <a class="header-anchor" href="#inspect-evidence-without-a-model-turn" aria-label="Permalink to &quot;Inspect evidence without a model turn&quot;">​</a></h2><p>Call the preparation tool directly:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> call</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> prepare_pr</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+import{_ as a,c as t,o as s,ag as i}from"./chunks/framework.CAZyNGu9.js";const u=JSON.parse('{"title":"Review prepared pull-request evidence","description":"Fetch a PR on the host, seed a trimmed diff-first workspace, and run a GitHub-read-only review through HTTP, Slack, or an eval.","frontmatter":{"title":"Review prepared pull-request evidence","description":"Fetch a PR on the host, seed a trimmed diff-first workspace, and run a GitHub-read-only review through HTTP, Slack, or an eval."},"headers":[],"relativePath":"example-agents/bugbot.md","filePath":"example-agents/bugbot.md"}'),r={name:"example-agents/bugbot.md"};function n(l,e,o,h,d,p){return s(),t("div",null,[...e[0]||(e[0]=[i(`<h1 id="review-prepared-pull-request-evidence" tabindex="-1">Review prepared pull-request evidence <a class="header-anchor" href="#review-prepared-pull-request-evidence" aria-label="Permalink to &quot;Review prepared pull-request evidence&quot;">​</a></h1><p>This GitHub-read-only reviewer uses host code to fetch the PR with <code>gh</code> and <code>git</code>, builds a trimmed <code>pr/</code> evidence tree, then hands that tree to the model. The model reads the diff, loads a review skill, and returns at most three high-confidence findings.</p><p>Use this example when the host should control evidence collection and the model shouldn&#39;t browse or mutate the source repository.</p><p><a href="./../../examples/bugbot/">Browse the current reviewer source.</a></p><h2 id="separate-evidence-preparation-from-review" tabindex="-1">Separate evidence preparation from review <a class="header-anchor" href="#separate-evidence-preparation-from-review" aria-label="Permalink to &quot;Separate evidence preparation from review&quot;">​</a></h2><p>The reviewer separates preparation from judgment:</p><ul><li>Host code owns GitHub and Git access.</li><li>A server tool turns untrusted PR input into bounded workspace files.</li><li>A custom channel seeds those files before the model starts.</li><li>An on-demand skill defines the review procedure and output contract.</li><li>The model returns chat text. No path posts a GitHub review.</li></ul><p>This architecture gives the model a purpose-built evidence package instead of a checkout.</p><h2 id="follow-a-review" tabindex="-1">Follow a review <a class="header-anchor" href="#follow-a-review" aria-label="Permalink to &quot;Follow a review&quot;">​</a></h2><p>The custom HTTP path runs this sequence:</p><ol><li><code>POST /v1/channels/review/</code> receives a PR reference.</li><li>The handler calls <code>prepare_pr</code> without a model turn.</li><li>Host code reads PR metadata and the unified diff.</li><li>It reuses a matching checkout, force-fetching the PR ref there when the commit is missing. Without a matching checkout, it uses a temporary bare cache.</li><li>It creates <code>pr/MANIFEST.md</code>, <code>pr/meta.json</code>, <code>pr/diff.patch</code>, and selected small files and rules.</li><li><code>send({ workspaceFiles })</code> creates the model session with that evidence.</li><li>The model reads the manifest and diff, then loads <code>pr-review</code>.</li><li>The channel returns session and playground URLs while the review streams.</li></ol><p>If a normal chat starts without evidence, the model can call <code>prepare_pr</code> mid-turn. That form writes the same files into the active session workspace.</p><h2 id="map-the-evidence-review-files" tabindex="-1">Map the evidence-review files <a class="header-anchor" href="#map-the-evidence-review-files" aria-label="Permalink to &quot;Map the evidence-review files&quot;">​</a></h2><table tabindex="0"><thead><tr><th>File</th><th>Purpose</th></tr></thead><tbody><tr><td><a href="../../examples/bugbot/agent/agent.ts"><code>agent/agent.ts</code></a></td><td>Selects the local runtime and model.</td></tr><tr><td><a href="./../../examples/bugbot/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Requires diff-first review and confines model work to <code>pr/</code>.</td></tr><tr><td><a href="../../examples/bugbot/agent/tools/prepare_pr.ts"><code>agent/tools/prepare_pr.ts</code></a></td><td>Exposes host preparation as a typed server tool.</td></tr><tr><td><a href="../../examples/bugbot/agent/lib/prepare-pr.ts"><code>agent/lib/prepare-pr.ts</code></a></td><td>Parses PR references, runs <code>gh</code> and <code>git</code>, and builds the evidence map.</td></tr><tr><td><a href="../../examples/bugbot/agent/channels/review.ts"><code>agent/channels/review.ts</code></a></td><td>Provides the loopback-only prepare-and-send HTTP route.</td></tr><tr><td><a href="../../examples/bugbot/agent/channels/slack.ts"><code>agent/channels/slack.ts</code></a></td><td>Extracts PR references and prepares evidence for mentions and direct messages.</td></tr><tr><td><a href="./../../examples/bugbot/agent/skills/pr-review.html"><code>agent/skills/pr-review.md</code></a></td><td>Sets finding limits, severities, and the machine-readable review format.</td></tr><tr><td><a href="../../examples/bugbot/evals/review/smoke.eval.ts"><code>evals/review/smoke.eval.ts</code></a></td><td>Seeds fake evidence and checks the review path without GitHub.</td></tr></tbody></table><p>There is no authored GitHub channel, MCP connection, subagent, schedule, hook, A/B experiment, approval, or custom storage.</p><h2 id="prepare-the-host" tabindex="-1">Prepare the host <a class="header-anchor" href="#prepare-the-host" aria-label="Permalink to &quot;Prepare the host&quot;">​</a></h2><p>You need:</p><ul><li>Node 22.13 or newer.</li><li>An agent-runtime credential for model turns and account-linked Slack.</li><li><code>gh</code> and <code>git</code> on <code>PATH</code>.</li><li><code>gh</code> access to the target PR.</li><li>Network access to GitHub and a writable temporary directory.</li></ul><p>The preparer can prefer a configured local checkout. Its <code>origin</code> must match the target repository. Otherwise the reviewer uses its bare cache. It never checks out the PR into the serve host&#39;s working tree.</p><h2 id="validate-the-surface" tabindex="-1">Validate the surface <a class="header-anchor" href="#validate-the-surface" aria-label="Permalink to &quot;Validate the surface&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/bugbot</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> info</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/bugbot</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p>The manifest should show one server tool, one skill, and two authored channels.</p><h2 id="inspect-evidence-without-a-model-turn" tabindex="-1">Inspect evidence without a model turn <a class="header-anchor" href="#inspect-evidence-without-a-model-turn" aria-label="Permalink to &quot;Inspect evidence without a model turn&quot;">​</a></h2><p>Call the preparation tool directly:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> call</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> prepare_pr</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/bugbot</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
-<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --input</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &#39;{&quot;pr&quot;:&quot;https://github.com/owner/repo/pull/123&quot;}&#39;</span></span></code></pre></div><p>Direct tool calls use a scratch workspace removed after the call. <code>prepare_pr</code> detects this path and returns the complete file map in its result. In a model session, it writes the files and returns a smaller summary.</p><p>The evidence builder applies explicit limits:</p><table tabindex="0"><thead><tr><th>Evidence</th><th>Limit</th></tr></thead><tbody><tr><td>Post-change file</td><td>12,000 characters</td></tr><tr><td>One rule file</td><td>8,000 characters</td></tr><tr><td>Combined rules</td><td>12,000 characters</td></tr><tr><td>PR body in metadata</td><td>2,000 characters</td></tr></tbody></table><p>Large files remain visible in <code>diff.patch</code>. The manifest records which full files or rules were omitted.</p><p>The per-file limits aren&#39;t an aggregate context cap. Every changed file below 12,000 characters can be included. The diff command has a 12 MiB output buffer; a larger diff fails preparation instead of being truncated.</p><h2 id="run-the-http-review-path" tabindex="-1">Run the HTTP review path <a class="header-anchor" href="#run-the-http-review-path" aria-label="Permalink to &quot;Run the HTTP review path&quot;">​</a></h2><p>Start the server:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/bugbot</span></span></code></pre></div><p>From another terminal:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">curl</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -s</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -X</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> POST</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --input</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &#39;{&quot;pr&quot;:&quot;https://github.com/owner/repo/pull/123&quot;}&#39;</span></span></code></pre></div><p>Direct tool calls use a scratch workspace removed after the call. <code>prepare_pr</code> detects this path and returns the complete file map in its result. In a model session, it writes the files and returns a smaller summary.</p><p>The evidence builder applies explicit limits:</p><table tabindex="0"><thead><tr><th>Evidence</th><th>Limit</th></tr></thead><tbody><tr><td>Post-change file</td><td>12,000 characters</td></tr><tr><td>One rule file</td><td>8,000 characters</td></tr><tr><td>Combined rules</td><td>12,000 characters</td></tr><tr><td>PR body in metadata</td><td>2,000 characters</td></tr></tbody></table><p>Large files remain visible in <code>diff.patch</code>. The manifest records which full files or rules were omitted.</p><p>The per-file limits aren&#39;t an aggregate context cap. Every changed file below 12,000 characters can be included. The diff command has a 12 MiB output buffer; a larger diff fails preparation instead of being truncated.</p><h2 id="run-the-http-review-path" tabindex="-1">Run the HTTP review path <a class="header-anchor" href="#run-the-http-review-path" aria-label="Permalink to &quot;Run the HTTP review path&quot;">​</a></h2><p>Start the server:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/bugbot</span></span></code></pre></div><p>From another terminal:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">curl</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -s</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -X</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> POST</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">  http://127.0.0.1:3000/bugbot/v1/channels/review/</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  -H</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &#39;content-type: application/json&#39;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  -d</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &#39;{&quot;pr&quot;:&quot;https://github.com/owner/repo/pull/123&quot;}&#39;</span></span></code></pre></div><p>The route returns <code>status: &quot;started&quot;</code>, a continuation token, and session and playground URLs. Open the session URL to watch the model read the evidence and produce findings.</p><p>The channel declares <code>localDevStrict()</code>. Direct loopback callers can use it. Proxy-forwarding headers and non-loopback hosts are rejected.</p><p>Send a follow-up by passing the returned key:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">curl</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -s</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -X</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> POST</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">  http://127.0.0.1:3000/bugbot/v1/channels/review/</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  -H</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &#39;content-type: application/json&#39;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
-<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  -d</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &#39;{&quot;pr&quot;:&quot;owner/repo#123&quot;,&quot;key&quot;:&quot;&lt;continuation-token&gt;&quot;}&#39;</span></span></code></pre></div><p>The follow-up resumes the session without fetching a new evidence tree.</p><h2 id="run-the-slack-path" tabindex="-1">Run the Slack path <a class="header-anchor" href="#run-the-slack-path" aria-label="Permalink to &quot;Run the Slack path&quot;">​</a></h2><p>The account-linked Slack channel handles review-bot mentions and direct messages:</p><blockquote><p>Review <a href="https://github.com/owner/repo/pull/123" target="_blank" rel="noreferrer">https://github.com/owner/repo/pull/123</a></p></blockquote><p>Slack handlers don&#39;t receive the channel <code>callTool</code> helper. This example calls the shared <code>preparePrReview</code> host function, then returns <code>workspaceFiles</code> in the Slack message preparation result. The model sees the same evidence and prompt as the HTTP path.</p><p>If a message contains no PR reference, the handler asks for one. Thread follow-ups keep the same session.</p><h2 id="see-how-the-skill-constrains-review" tabindex="-1">See how the skill constrains review <a class="header-anchor" href="#see-how-the-skill-constrains-review" aria-label="Permalink to &quot;See how the skill constrains review&quot;">​</a></h2><p><code>pr-review.md</code> tells the model to:</p><ul><li>read the manifest and unified diff first,</li><li>open at most one supporting file or rules file when a hunk is ambiguous,</li><li>avoid shell, network, <code>gh</code>, and <code>git</code>,</li><li>report no more than three findings,</li><li>keep each description under 120 words, and</li><li>emit the machine-readable review contract.</li></ul><p>The root instructions set the evidence boundary. The skill holds the reusable review procedure. Keeping those roles separate lets another agent reuse the same skill with different intake channels.</p><h2 id="run-the-fixture-backed-eval" tabindex="-1">Run the fixture-backed eval <a class="header-anchor" href="#run-the-fixture-backed-eval" aria-label="Permalink to &quot;Run the fixture-backed eval&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/bugbot</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agentkit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/bugbot</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> review/smoke</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p>The eval constructs a <code>PreparedPrReview</code>, seeds its file map through <code>workspaceFiles</code>, and checks for at least one read call with no shell call. It doesn&#39;t assert which evidence file was read or whether the skill loaded. It accepts either a formatted review or a clean result.</p><p>This case tests review behavior without GitHub credentials or network data. Add fixtures with reachable bugs when you need stricter location and severity checks.</p><h2 id="keep-the-side-effect-boundary-clear" tabindex="-1">Keep the side-effect boundary clear <a class="header-anchor" href="#keep-the-side-effect-boundary-clear" aria-label="Permalink to &quot;Keep the side-effect boundary clear&quot;">​</a></h2><p>The reviewer makes no remote GitHub writes. It doesn&#39;t author a GitHub channel and doesn&#39;t call a review API. Host preparation does write session evidence and force-update <code>refs/pull/&lt;N&gt;/head</code> in either its bare cache or a matching local checkout when the commit is missing. Every result ends with a note saying no GitHub review was posted.</p><p>If you add publishing later, keep it in a separate tool. This preserves a read-only preparation and review path safe to run in evals.</p><h2 id="reuse-the-evidence-handoff" tabindex="-1">Reuse the evidence handoff <a class="header-anchor" href="#reuse-the-evidence-handoff" aria-label="Permalink to &quot;Reuse the evidence handoff&quot;">​</a></h2><p>Use host-prepared workspaces when:</p><ul><li>external APIs should stay off the model&#39;s tool surface,</li><li>context needs hard size limits,</li><li>the model should inspect a snapshot instead of a live checkout, or</li><li>several channels need the same preparation.</li></ul><p>Return <code>workspaceFiles</code> from direct host preparation, write into <code>ctx.workspaceDir</code> for mid-turn recovery, and encode the reading order in both the manifest and a skill.</p><h2 id="where-to-go-next" tabindex="-1">Where to go next <a class="header-anchor" href="#where-to-go-next" aria-label="Permalink to &quot;Where to go next&quot;">​</a></h2><ul><li><a href="./../guides/webhooks.html">Webhooks and custom channels</a></li><li><a href="./../reference/tools.html">Tools</a></li><li><a href="./../reference/skills.html">Skills</a></li><li><a href="./../guides/slack.html">Slack</a></li><li><a href="./../evals.html">Evals</a></li></ul>`,62)])])}const k=a(r,[["render",n]]);export{u as __pageData,k as default};
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  -d</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &#39;{&quot;pr&quot;:&quot;owner/repo#123&quot;,&quot;key&quot;:&quot;&lt;continuation-token&gt;&quot;}&#39;</span></span></code></pre></div><p>The follow-up resumes the session without fetching a new evidence tree.</p><h2 id="run-the-slack-path" tabindex="-1">Run the Slack path <a class="header-anchor" href="#run-the-slack-path" aria-label="Permalink to &quot;Run the Slack path&quot;">​</a></h2><p>The account-linked Slack channel handles review-bot mentions and direct messages:</p><blockquote><p>Review <a href="https://github.com/owner/repo/pull/123" target="_blank" rel="noreferrer">https://github.com/owner/repo/pull/123</a></p></blockquote><p>Slack handlers don&#39;t receive the channel <code>callTool</code> helper. This example calls the shared <code>preparePrReview</code> host function, then returns <code>workspaceFiles</code> in the Slack message preparation result. The model sees the same evidence and prompt as the HTTP path.</p><p>If a message contains no PR reference, the handler asks for one. Thread follow-ups keep the same session.</p><h2 id="see-how-the-skill-constrains-review" tabindex="-1">See how the skill constrains review <a class="header-anchor" href="#see-how-the-skill-constrains-review" aria-label="Permalink to &quot;See how the skill constrains review&quot;">​</a></h2><p><code>pr-review.md</code> tells the model to:</p><ul><li>read the manifest and unified diff first,</li><li>open at most one supporting file or rules file when a hunk is ambiguous,</li><li>avoid shell, network, <code>gh</code>, and <code>git</code>,</li><li>report no more than three findings,</li><li>keep each description under 120 words, and</li><li>emit the machine-readable review contract.</li></ul><p>The root instructions set the evidence boundary. The skill holds the reusable review procedure. Keeping those roles separate lets another agent reuse the same skill with different intake channels.</p><h2 id="run-the-fixture-backed-eval" tabindex="-1">Run the fixture-backed eval <a class="header-anchor" href="#run-the-fixture-backed-eval" aria-label="Permalink to &quot;Run the fixture-backed eval&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/bugbot</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/bugbot</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> review/smoke</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p>The eval constructs a <code>PreparedPrReview</code>, seeds its file map through <code>workspaceFiles</code>, and checks for at least one read call with no shell call. It doesn&#39;t assert which evidence file was read or whether the skill loaded. It accepts either a formatted review or a clean result.</p><p>This case tests review behavior without GitHub credentials or network data. Add fixtures with reachable bugs when you need stricter location and severity checks.</p><h2 id="keep-the-side-effect-boundary-clear" tabindex="-1">Keep the side-effect boundary clear <a class="header-anchor" href="#keep-the-side-effect-boundary-clear" aria-label="Permalink to &quot;Keep the side-effect boundary clear&quot;">​</a></h2><p>The reviewer makes no remote GitHub writes. It doesn&#39;t author a GitHub channel and doesn&#39;t call a review API. Host preparation does write session evidence and force-update <code>refs/pull/&lt;N&gt;/head</code> in either its bare cache or a matching local checkout when the commit is missing. Every result ends with a note saying no GitHub review was posted.</p><p>If you add publishing later, keep it in a separate tool. This preserves a read-only preparation and review path safe to run in evals.</p><h2 id="reuse-the-evidence-handoff" tabindex="-1">Reuse the evidence handoff <a class="header-anchor" href="#reuse-the-evidence-handoff" aria-label="Permalink to &quot;Reuse the evidence handoff&quot;">​</a></h2><p>Use host-prepared workspaces when:</p><ul><li>external APIs should stay off the model&#39;s tool surface,</li><li>context needs hard size limits,</li><li>the model should inspect a snapshot instead of a live checkout, or</li><li>several channels need the same preparation.</li></ul><p>Return <code>workspaceFiles</code> from direct host preparation, write into <code>ctx.workspaceDir</code> for mid-turn recovery, and encode the reading order in both the manifest and a skill.</p><h2 id="where-to-go-next" tabindex="-1">Where to go next <a class="header-anchor" href="#where-to-go-next" aria-label="Permalink to &quot;Where to go next&quot;">​</a></h2><ul><li><a href="./../guides/webhooks.html">Webhooks and custom channels</a></li><li><a href="./../reference/tools.html">Tools</a></li><li><a href="./../reference/skills.html">Skills</a></li><li><a href="./../guides/slack.html">Slack</a></li><li><a href="./../evals.html">Evals</a></li></ul>`,62)])])}const k=a(r,[["render",n]]);export{u as __pageData,k as default};