@cursor/july 0.1.91 → 0.1.93

package/dist/docs/assets/{evals.md.a0SMN6r9.js → evals.md.lfJoEVc8.js}

@@ -1,4 +1,4 @@
-import{_ as e,c as a,o as i,ag as t}from"./chunks/framework.BCISBCiQ.js";const k=JSON.parse('{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agent-sdk eval, and use them as regression checks.","frontmatter":{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agent-sdk eval, and use them as regression checks."},"headers":[],"relativePath":"evals.md","filePath":"evals.md"}'),n={name:"evals.md"};function l(d,s,o,h,r,p){return i(),a("div",null,[...s[0]||(s[0]=[t(`<h1 id="evals" tabindex="-1">Evals <a class="header-anchor" href="#evals" aria-label="Permalink to &quot;Evals&quot;">​</a></h1><p>An eval is a repeatable check that runs your agent against a fixed input and gates the recorded trajectory: the run completed, the right tool ran, the reply has the right shape. Evals are how you know a prompt tweak helped, a refactor didn&#39;t regress the agent, and last month&#39;s fix is still holding.</p><p>Evals exercise the same surface your users hit. The runner starts (or targets) a real agent server, drives sessions over the public API, and grades what comes back. A passing eval means the agent started, accepted a message, and did what you asserted.</p><div class="note custom-block github-alert"><p class="custom-block-title">NOTE</p><p>Import paths here use <code>@cursor/july/evals</code>. On projects still using <code>@anysphere/agent-serve</code>, swap the import and run <code>agent-serve eval</code>. See <a href="/docs/#run-the-cli">Run the CLI</a> for the full rename table.</p></div><h2 id="define-evals-with-defineeval" tabindex="-1">Define evals with <code>defineEval</code> <a class="header-anchor" href="#define-evals-with-defineeval" aria-label="Permalink to &quot;Define evals with \`defineEval\`&quot;">​</a></h2><p>The Agent SDK discovers evals under the project-root <code>evals/</code> directory, in <code>.eval.ts</code> or <code>.eval.js</code> files. That&#39;s a sibling of <code>agent/</code>, never inside it (<code>agent/evals/</code> is silently ignored). TypeScript is the normal authoring format.</p><p>The file path is the eval&#39;s identity, so you don&#39;t author an id. Directories group related evals: <code>evals/builds/api.eval.ts</code> becomes id <code>builds/api</code>. An <code>index</code> filename collapses to its directory, so <code>evals/builds/index.eval.ts</code> becomes <code>builds</code>.</p><p>An eval is a single <code>async test(t)</code>. You drive the agent with <code>t</code> and assert on the run with the same <code>t</code>:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/readiness.eval.ts</span></span>
+import{_ as e,c as i,o as a,ag as t}from"./chunks/framework.BCISBCiQ.js";const k=JSON.parse('{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agent-sdk eval, and use them as regression checks.","frontmatter":{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agent-sdk eval, and use them as regression checks."},"headers":[],"relativePath":"evals.md","filePath":"evals.md"}'),n={name:"evals.md"};function l(d,s,h,o,r,p){return a(),i("div",null,[...s[0]||(s[0]=[t(`<h1 id="evals" tabindex="-1">Evals <a class="header-anchor" href="#evals" aria-label="Permalink to &quot;Evals&quot;">​</a></h1><p>An eval is a repeatable check that runs your agent against a fixed input and gates the recorded trajectory: the run completed, the right tool ran, the reply has the right shape. Evals are how you know a prompt tweak helped, a refactor didn&#39;t regress the agent, and last month&#39;s fix is still holding.</p><p>Evals exercise the same surface your users hit. The runner starts (or targets) a real agent server, drives sessions over the public API, and grades what comes back. A passing eval means the agent started, accepted a message, and did what you asserted.</p><h2 id="define-evals-with-defineeval" tabindex="-1">Define evals with <code>defineEval</code> <a class="header-anchor" href="#define-evals-with-defineeval" aria-label="Permalink to &quot;Define evals with \`defineEval\`&quot;">​</a></h2><p>The Agent SDK discovers evals under the project-root <code>evals/</code> directory, in <code>.eval.ts</code> or <code>.eval.js</code> files. That&#39;s a sibling of <code>agent/</code>, never inside it (<code>agent/evals/</code> is silently ignored). TypeScript is the normal authoring format.</p><p>The file path is the eval&#39;s identity, so you don&#39;t author an id. Directories group related evals: <code>evals/builds/api.eval.ts</code> becomes id <code>builds/api</code>. An <code>index</code> filename collapses to its directory, so <code>evals/builds/index.eval.ts</code> becomes <code>builds</code>.</p><p>An eval is a single <code>async test(t)</code>. You drive the agent with <code>t</code> and assert on the run with the same <code>t</code>:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/readiness.eval.ts</span></span>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineEval, includes } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/evals&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
 <span class="line"></span>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEval</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
@@ -47,8 +47,8 @@ import{_ as e,c as a,o as i,ag as t}from"./chunks/framework.BCISBCiQ.js";const k
 <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">  // timeoutMs: 180_000,</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> // optional project-wide default</span></span>
 <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">  // judge: { model: &quot;...&quot; },</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> // default judge model for t.judge.*</span></span>
 <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">  // reporters: [],</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> // destinations that observe every case</span></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">  // maxPlaygroundRuns: 50,</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> // playground /v1/dev/evals history only (default 20)</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p>The timeout order is case or file <code>timeoutMs</code>, CLI <code>--timeout-ms</code>, project config <code>timeoutMs</code>, then the 180-second runner default.</p><p>The optional fields:</p><table tabindex="0"><thead><tr><th>Option</th><th>Default</th><th>Meaning</th></tr></thead><tbody><tr><td><code>timeoutMs</code></td><td><code>180_000</code></td><td>Project-wide per-case timeout</td></tr><tr><td><code>judge</code></td><td>unset</td><td>Default judge model for <code>t.judge.*</code>; see <a href="#judge-free-form-output">Judge free-form output</a></td></tr><tr><td><code>reporters</code></td><td>unset</td><td>Destinations that observe every case; <code>--skip-report</code> suppresses them</td></tr><tr><td><code>maxPlaygroundRuns</code></td><td><code>20</code></td><td>Max batches in the playground / <code>/v1/dev/evals*</code> history (not CLI <code>eval</code>)</td></tr></tbody></table><p>Reporters come from <code>@cursor/july/evals/reporters</code>: <code>JUnit</code> writes a JUnit XML file for CI, <code>Artifacts</code> writes per-case files, and <code>combineReporters</code> merges several into one (<code>renderJUnitXml</code> renders the XML for a custom destination). A file or case can add its own <code>reporters</code> on top of the config list.</p><p>Playground batches survive restarts whenever <code>agent/storage.ts</code> exists with an <code>evals</code> table or a KV core providing <code>delete</code> and <code>list</code> (the table is derived over the core); see <a href="./storage.html#eval-and-a-b-tables">Storage</a>. Without storage they live in process memory and disappear when <code>serve</code> exits — navigating away and back still works while the process is up.</p><h2 id="drive-and-assert-with-t" tabindex="-1">Drive and assert with <code>t</code> <a class="header-anchor" href="#drive-and-assert-with-t" aria-label="Permalink to &quot;Drive and assert with \`t\`&quot;">​</a></h2><p><code>t</code> is both the driver and the assertion surface. You write ordinary control flow, sending turns and asserting inline.</p><p>Drive the agent with <code>t.send(message, options?)</code>. It runs one turn and waits for the session to park or fail. Multiple sends in one case share the session, which is how you write multi-turn evals.</p><p>Each <code>t.send</code> resolves to a turn result with <code>message</code>, <code>sessionId</code>, <code>events</code>, <code>toolCalls</code>, <code>ok</code>, and <code>index</code>. The turn carries the same assertion vocabulary as <code>t</code>, scoped to that turn, so you can grade an intermediate turn before the next send overwrites <code>t.reply</code>. <code>turn.expectOk()</code> throws when the turn failed, for later steps that depend on it.</p><p>Read the full case state with <code>t.reply</code> (the last assistant text), <code>t.events</code> (every captured session event across turns), <code>t.turns</code> (settled turns, oldest first), and <code>t.sessionId</code>. <code>t.signal</code> aborts when the case hits its timeout; pass it to your own async work.</p><p>Assert with the gates:</p><table tabindex="0"><thead><tr><th>Gate</th><th>Checks</th></tr></thead><tbody><tr><td><code>t.succeeded()</code></td><td>the run did not fail and is not parked on an unanswered approval</td></tr><tr><td><code>t.parked()</code></td><td>the run cleanly parked on an unanswered approval request</td></tr><tr><td><code>t.messageIncludes(token)</code></td><td>the joined assistant text matches a string or <code>RegExp</code></td></tr><tr><td><code>t.calledTool(name, matcher?)</code></td><td>a matching call to <code>name</code> happened</td></tr><tr><td><code>t.notCalledTool(name)</code></td><td>no request for <code>name</code>, in any lifecycle state</td></tr><tr><td><code>t.loadedSkill(name)</code></td><td>the agent opened the skill&#39;s <code>SKILL.md</code> (read, grep, or shell <code>cat</code>)</td></tr><tr><td><code>t.toolOrder(names)</code></td><td>tool requests appear in this relative order (extra calls allowed)</td></tr><tr><td><code>t.usedNoTools()</code></td><td>no tool calls at all</td></tr><tr><td><code>t.maxToolCalls(max)</code></td><td>at most <code>max</code> tool calls</td></tr><tr><td><code>t.noFailedActions()</code></td><td>no tool call reported an error</td></tr><tr><td><code>t.calledSubagent(name, matcher?)</code></td><td>a matching subagent delegation happened</td></tr><tr><td><code>t.taggedArtifact(kind?, predicate?)</code></td><td>at least one <a href="./reference/artifacts.html">artifact</a> was tagged</td></tr><tr><td><code>t.event(type, matcher?)</code></td><td>at least one matching event of <code>type</code> occurred</td></tr><tr><td><code>t.notEvent(type, matcher?)</code></td><td>no matching event of <code>type</code> occurred</td></tr><tr><td><code>t.eventOrder(matchers)</code></td><td>matching event groups occur in this relative order</td></tr><tr><td><code>t.eventsSatisfy(label, predicate)</code></td><td>your predicate over the typed event stream</td></tr><tr><td><code>t.check(value, expectation)</code></td><td>any value, against a builder</td></tr><tr><td><code>t.score(name, value)</code></td><td>records a 0–1 score you computed; soft until you add a bar</td></tr><tr><td><code>t.requireToolCall(name, matcher?)</code></td><td>gates on a matching call and returns it, so later code can read its input and output</td></tr><tr><td><code>t.requireInputRequest(filter?)</code></td><td>gates on exactly one pending approval request and returns it</td></tr></tbody></table><p>Every gate returns a handle: <code>.soft()</code> demotes it to tracked-only, <code>.atLeast(0.7)</code> adds a soft score bar, and <code>.gate(0.8)</code> promotes a scored assertion into a hard gate.</p><p>With no matcher, <code>calledTool</code> is request-based: a requested call counts even when its result has not arrived. Pass <code>t.calledTool(&quot;inspect_pr&quot;, { status: &quot;completed&quot; })</code> to require the call to return. <code>input</code>, <code>output</code>, and <code>count</code> matcher fields accept a literal, a <code>RegExp</code>, or a predicate.</p><p>The expectation builders are <code>includes(string | RegExp)</code>, <code>equals(value)</code>, <code>matches(schema)</code>, <code>similarity(expected)</code>, and <code>satisfies(predicate, label)</code>. <code>includes</code> stringifies its input, <code>equals</code> compares values deeply, <code>matches</code> validates against a Standard Schema (or anything with <code>safeParse</code>, like Zod), <code>similarity</code> scores normalized text similarity, and <code>satisfies</code> runs your predicate. The plain function <code>normalizedSimilarity(actual, expected)</code> returns the same 0–1 score for use with <code>t.score</code>.</p><p>A few more context members shape a case: <code>t.require(value, expectation)</code> records a gate and stops the test body when it fails, without a duplicate execution error. <code>t.skip(reason)</code> ends the case as skipped (reported separately, never changes the exit code; call it before sending messages). <code>t.metric(name, value)</code> records a structured score for the playground case card. <code>t.log(message)</code> records a debug line for the CLI and playground result.</p><p>Three <code>t.send</code> options apply on session create (first <code>t.send</code> only):</p><ul><li><code>workspaceFiles</code> — <code>{ path: contents }</code>, seeded into the local session workspace. Prefer this over machine-local paths.</li><li><code>workspaceDir</code> — absolute harness cwd (local runtime).</li><li><code>cloud</code> — per-session cloud options merged over the agent&#39;s static <code>cloud</code> config (repos / env / …). Use a pinned <code>repos</code> override to attach a fixture repo for cloud evals without putting it on the agent&#39;s default <code>cloud.repos</code>. Cloud ignores <code>workspaceFiles</code> seeds.</li></ul><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> toolResults</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.events.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">filter</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">((</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">e</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=&gt;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> e.type </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">===</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;action.result&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
+<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">  // maxPlaygroundRuns: 50,</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> // playground history only (default 20)</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p>The timeout order is case or file <code>timeoutMs</code>, CLI <code>--timeout-ms</code>, project config <code>timeoutMs</code>, then the 180-second runner default.</p><p>The optional fields:</p><table tabindex="0"><thead><tr><th>Option</th><th>Default</th><th>Meaning</th></tr></thead><tbody><tr><td><code>timeoutMs</code></td><td><code>180_000</code></td><td>Project-wide per-case timeout</td></tr><tr><td><code>judge</code></td><td>unset</td><td>Default judge model for <code>t.judge.*</code>; see <a href="#judge-free-form-output">Judge free-form output</a></td></tr><tr><td><code>reporters</code></td><td>unset</td><td>Destinations that observe every case; <code>--skip-report</code> suppresses them</td></tr><tr><td><code>maxPlaygroundRuns</code></td><td><code>20</code></td><td>Max batches in the playground / <code>/v1/dev/evals*</code> history (not CLI <code>eval</code>). Hard-capped at 500.</td></tr></tbody></table><p>Reporters come from <code>@cursor/july/evals/reporters</code>: <code>JUnit</code> writes a JUnit XML file for CI, <code>Artifacts</code> writes per-case files, and <code>combineReporters</code> merges several into one (<code>renderJUnitXml</code> renders the XML for a custom destination). A file or case can add its own <code>reporters</code> on top of the config list.</p><p>Playground batches survive restarts whenever <code>agent/storage.ts</code> exists with an <code>evals</code> table or a KV core providing <code>delete</code> and <code>list</code> (the table is derived over the core); see <a href="./storage.html#eval-and-a-b-tables">Storage</a>. Without storage they live in process memory and disappear when <code>serve</code> exits. Navigating away and back still works while the process is up.</p><h2 id="drive-and-assert-with-t" tabindex="-1">Drive and assert with <code>t</code> <a class="header-anchor" href="#drive-and-assert-with-t" aria-label="Permalink to &quot;Drive and assert with \`t\`&quot;">​</a></h2><p><code>t</code> is both the driver and the assertion surface. You write ordinary control flow, sending turns and asserting inline.</p><p>Drive the agent with <code>t.send(message, options?)</code>. It runs one turn and waits for the session to park or fail. Multiple sends in one case share the session, which is how you write multi-turn evals.</p><p>Each <code>t.send</code> resolves to a turn result with <code>message</code>, <code>sessionId</code>, <code>events</code>, <code>toolCalls</code>, <code>ok</code>, and <code>index</code>. The turn carries the same assertion vocabulary as <code>t</code>, scoped to that turn, so you can grade an intermediate turn before the next send overwrites <code>t.reply</code>. <code>turn.expectOk()</code> throws when the turn failed, for later steps that depend on it.</p><p>Read the full case state with <code>t.reply</code> (the last assistant text), <code>t.events</code> (session events captured so far), <code>t.turns</code> (settled turns, oldest first), and <code>t.sessionId</code>. <code>t.signal</code> aborts when the case hits its timeout; pass it to your own async work.</p><p>Assert with the gates:</p><table tabindex="0"><thead><tr><th>Gate</th><th>Checks</th></tr></thead><tbody><tr><td><code>t.succeeded()</code></td><td>the run did not fail and is not parked on an unanswered approval</td></tr><tr><td><code>t.parked()</code></td><td>the run cleanly parked on an unanswered approval request</td></tr><tr><td><code>t.messageIncludes(token)</code></td><td>the joined assistant text matches a string or <code>RegExp</code></td></tr><tr><td><code>t.calledTool(name, matcher?)</code></td><td>a matching call to <code>name</code> happened</td></tr><tr><td><code>t.notCalledTool(name)</code></td><td>no request for <code>name</code>, in any lifecycle state</td></tr><tr><td><code>t.loadedSkill(name)</code></td><td>the agent opened the skill&#39;s <code>SKILL.md</code> (read, grep, or shell <code>cat</code>)</td></tr><tr><td><code>t.toolOrder(names)</code></td><td>tool requests appear in this relative order (extra calls allowed)</td></tr><tr><td><code>t.usedNoTools()</code></td><td>no tool calls at all</td></tr><tr><td><code>t.maxToolCalls(max)</code></td><td>at most <code>max</code> tool calls</td></tr><tr><td><code>t.noFailedActions()</code></td><td>no tool call reported an error</td></tr><tr><td><code>t.calledSubagent(name, matcher?)</code></td><td>a matching subagent delegation happened</td></tr><tr><td><code>t.taggedArtifact(kind?, predicate?)</code></td><td>at least one <a href="./reference/artifacts.html">artifact</a> was tagged</td></tr><tr><td><code>t.event(type, matcher?)</code></td><td>at least one matching event of <code>type</code> occurred</td></tr><tr><td><code>t.notEvent(type, matcher?)</code></td><td>no matching event of <code>type</code> occurred</td></tr><tr><td><code>t.eventOrder(matchers)</code></td><td>matching event groups occur in this relative order</td></tr><tr><td><code>t.eventsSatisfy(label, predicate)</code></td><td>your predicate over the typed event stream</td></tr><tr><td><code>t.check(value, expectation)</code></td><td>any value, against a builder</td></tr><tr><td><code>t.score(name, value)</code></td><td>records a 0–1 score you computed; soft until you add a bar</td></tr><tr><td><code>t.requireToolCall(name, matcher?)</code></td><td>gates on a matching call and returns it, so later code can read its input and output</td></tr><tr><td><code>t.requireInputRequest(filter?)</code></td><td>gates on exactly one pending approval request and returns it</td></tr></tbody></table><p>Every gate returns a handle: <code>.soft()</code> demotes it to tracked-only, <code>.atLeast(0.7)</code> adds a soft score bar, and <code>.gate(0.8)</code> promotes a scored assertion into a hard gate.</p><p>With no matcher, <code>calledTool</code> is request-based: a requested call counts even when its result has not arrived. Pass <code>t.calledTool(&quot;inspect_pr&quot;, { status: &quot;completed&quot; })</code> to require the call to return. <code>input</code>, <code>output</code>, and <code>count</code> matcher fields accept a literal, a <code>RegExp</code>, or a predicate.</p><p>The expectation builders are <code>includes(string | RegExp)</code>, <code>equals(value)</code>, <code>matches(schema)</code>, <code>similarity(expected)</code>, and <code>satisfies(predicate, label)</code>. <code>includes</code> stringifies its input, <code>equals</code> compares values deeply, <code>matches</code> validates against a Standard Schema (or anything with <code>safeParse</code>, like Zod), <code>similarity</code> scores normalized text similarity, and <code>satisfies</code> runs your predicate. The plain function <code>normalizedSimilarity(actual, expected)</code> returns the same 0–1 score for use with <code>t.score</code>.</p><p>A few more context members shape a case: <code>t.require(value, expectation)</code> records a gate and stops the test body when it fails, without a duplicate execution error. <code>t.skip(reason)</code> ends the case as skipped (reported separately, never changes the exit code; call it before sending messages). <code>t.metric(name, value)</code> records a structured score for the playground case card. <code>t.log(message)</code> records a debug line for the CLI and playground result.</p><p>Three <code>t.send</code> options apply on session create (first <code>t.send</code> only):</p><ul><li><code>workspaceFiles</code>: <code>{ path: contents }</code>, seeded into the local session workspace. Prefer this over machine-local paths.</li><li><code>workspaceDir</code>: absolute harness cwd (local runtime).</li><li><code>cloud</code>: per-session cloud options merged over the agent&#39;s static <code>cloud</code> config (repos / env / …). Use a pinned <code>repos</code> override to attach a fixture repo for cloud evals without putting it on the agent&#39;s default <code>cloud.repos</code>. Cloud ignores <code>workspaceFiles</code> seeds.</li></ul><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> toolResults</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.events.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">filter</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">((</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">e</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=&gt;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> e.type </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">===</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;action.result&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  toolResults.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">length</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">  satisfies</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">((</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">n</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=&gt;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> (n </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">as</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> number</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">&lt;=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> 4</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;at most 4 tool calls&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span>
@@ -58,7 +58,7 @@ import{_ as e,c as a,o as i,ag as t}from"./chunks/framework.BCISBCiQ.js";const k
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> builds</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> search</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                  # several ids or prefixes</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> smoke</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> pull-request</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # any matching tag</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --no-stream</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">             # machine-readable results</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --verbose</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                      # logs + reply snippets</span></span></code></pre></div><p>Id filters use OR semantics. Each filter selects an exact id and its descendants. For example, <code>builds</code> selects <code>builds</code>, <code>builds/checkout</code>, and every other case below that path. Repeated tags also use OR semantics. When you provide both ids and tags, a case must match both groups.</p><p><code>eval</code> boots an ephemeral server on port 0 with a temp state root outside the project, so cases don&#39;t inherit ambient monorepo rules and don&#39;t pollute <code>.agent-serve/</code>. Point <code>--url</code> at a running server to eval a live agent instead:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --verbose</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">                      # logs + reply snippets</span></span></code></pre></div><p>Id filters use OR semantics. Each filter selects an exact id and its descendants. For example, <code>builds</code> selects <code>builds</code>, <code>builds/checkout</code>, and every other case below that path. Repeated tags also use OR semantics. When you provide both ids and tags, a case must match both groups.</p><p><code>eval</code> boots an ephemeral server on port 0 with a temp state root outside the project, so cases don&#39;t inherit ambient monorepo rules and don&#39;t write into the project state directory. Point <code>--url</code> at a running server to eval a live agent instead:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --url</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> http://127.0.0.1:3000/weather-agent</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --bearer-token</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$AGENT_TOKEN</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;</span></span></code></pre></div><p>The eval definitions still come from <code>--dir</code>; <code>--url</code> only changes the agent that receives the turns. For a locally mounted multi-agent directory, <code>--slug weather-agent</code> chooses the target. Use <code>--state-root</code> to keep ephemeral session state at a chosen path, <code>--timeout-ms</code> to override the project timeout, and <code>--no-stream</code> to keep live progress off stderr. A TTY streams turn progress by default. <code>--verbose</code> still writes <code>t.log</code> lines to stderr and adds reply snippets to text results.</p><p>Model turns need a Cursor credential from <code>agent-sdk login</code> or <code>CURSOR_API_KEY</code>.</p><p>See <a href="./reference/cli.html#eval">CLI: eval</a> for flags and exit codes.</p><h3 id="json-results" tabindex="-1">JSON results <a class="header-anchor" href="#json-results" aria-label="Permalink to &quot;JSON results&quot;">​</a></h3><p>Use <code>--json --no-stream</code> in scripts and CI. The top-level result carries the totals and one result per case:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;ok&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
@@ -76,10 +76,10 @@ import{_ as e,c as a,o as i,ag as t}from"./chunks/framework.BCISBCiQ.js";const k
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">      &quot;durationMs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">12340</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    }</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ]</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Each case result can also include <code>description</code>, <code>finalText</code>, <code>tools</code>, <code>error</code>, and tool arguments or output. This shape lets CI report the failed assertion without parsing terminal text.</p><h2 id="run-evals-in-the-playground" tabindex="-1">Run evals in the playground <a class="header-anchor" href="#run-evals-in-the-playground" aria-label="Permalink to &quot;Run evals in the playground&quot;">​</a></h2><p>Start the server with <code>--dev</code>, open the playground, and choose <strong>Evals</strong>. You can run every case or one case, watch progress, and open the resulting session trace.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> serve</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dev</span></span></code></pre></div><p>Playground runs target the live server instead of an ephemeral one. Their sessions appear in the session list. One eval batch can run at a time. Batches persist across restarts whenever <code>agent/storage.ts</code> provides an <code>evals</code> table or a KV core with <code>delete</code> and <code>list</code> (the table is derived over the core); without storage they are <strong>in-memory only</strong> (capped by <code>maxPlaygroundRuns</code>) — see <a href="./storage.html#eval-and-a-b-tables">Storage</a>.</p><p>The UI uses the playground eval routes (available without <code>--dev</code>): <code>GET /v1/dev/evals</code> lists datapoints and config (includes <code>maxPlaygroundRuns</code> / <code>durableRuns</code>), <code>GET /v1/dev/evals/runs</code> rehydrates recent batches after navigation, <code>POST /v1/dev/evals/runs</code> starts a batch (returns an <strong>Eval ID</strong> / <code>runId</code>), <code>GET /v1/dev/evals/runs/:runId</code> polls it, and <code>POST /v1/dev/evals/runs/:runId/cancel</code> cancels a running batch. See <a href="./reference/http-api.html#playground-eval-routes">Playground eval routes</a>. The start request returns <code>202</code> while cases run in the background. Poll until the snapshot status becomes <code>completed</code>, <code>failed</code>, or <code>cancelled</code>. Configuration errors appear on a failed snapshot.</p><p>On <code>--prod</code> / <code>--url</code>, the CLI prints the Eval ID as soon as the batch is accepted (and a Playground deep link with <code>?view=evals&amp;evalRunId=…</code>):</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> deepsec</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Each case result can also include <code>description</code>, <code>finalText</code>, <code>tools</code>, <code>error</code>, and tool arguments or output. This shape lets CI report the failed assertion without parsing terminal text.</p><h2 id="run-evals-in-the-playground" tabindex="-1">Run evals in the playground <a class="header-anchor" href="#run-evals-in-the-playground" aria-label="Permalink to &quot;Run evals in the playground&quot;">​</a></h2><p>Start the server, open the playground, and choose <strong>Evals</strong>. You can run every case or one case, watch progress, and open the resulting session trace. The Evals tab works on a normal <code>serve</code>.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> serve</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span></span></code></pre></div><p>Playground runs target the live server instead of an ephemeral one. Their sessions appear in the session list. One eval batch can run at a time. Persistence follows the rule under <a href="#configure-eval-runs">Configure eval runs</a>. See <a href="./reference/http-api.html#playground-eval-routes">Playground eval routes</a>. The start request returns <code>202</code> while cases run in the background. Poll until the snapshot status becomes <code>completed</code>, <code>failed</code>, or <code>cancelled</code>. Configuration errors appear on a failed snapshot.</p><p>On <code>--prod</code> / <code>--url</code>, the CLI prints the Eval ID as soon as the batch is accepted (and a Playground deep link with <code>?view=evals&amp;evalRunId=…</code>):</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> deepsec</span></span>
 <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Eval ID: evalrun_…</span></span>
 <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Cancel:  agent-sdk eval cancel evalrun_… --prod --slug vulnerability-scanner</span></span>
 <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Playground: https://…/playground?view=evals&amp;evalRunId=evalrun_…</span></span>
 <span class="line"></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> cancel</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> evalrun_…</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> status</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> evalrun_…</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span></span></code></pre></div><p>The Evals tab prefers the server’s in-flight batch (<code>activeRunId</code>) over a stale tab-local remembered id, so CLI / Slack kicks show up without an incognito window.</p><h2 id="what-good-cases-assert" tabindex="-1">What good cases assert <a class="header-anchor" href="#what-good-cases-assert" aria-label="Permalink to &quot;What good cases assert&quot;">​</a></h2><p>Gate decisions and shape, not prose. Model wording varies run to run. Tool choice, tool avoidance, and output structure are the stable contract.</p><ol><li><code>t.succeeded()</code>: always, first.</li><li>The tool decision: <code>calledTool</code> for the intended path, <code>notCalledTool</code> for the likely wrong alternative. The pair is stronger than either alone.</li><li>Output shape: a regex for the contract (<code>/ready|blocked/i</code>, a JSON marker, a findings-block fence), never exact sentences.</li><li>For structured output, parse <code>t.reply</code> and check fields with <code>satisfies</code> instead of substring-matching JSON.</li></ol><p>The common failure modes: asserting exact phrasing, packing more than about five gates into one case (split it), and cases that depend on live external state that drifts (pin the input; see fixtures).</p><h2 id="pick-fixtures-by-agent-type" tabindex="-1">Pick fixtures by agent type <a class="header-anchor" href="#pick-fixtures-by-agent-type" aria-label="Permalink to &quot;Pick fixtures by agent type&quot;">​</a></h2><p>The right fixture depends on the surface under test.</p><table tabindex="0"><thead><tr><th>Agent surface</th><th>Fixture</th></tr></thead><tbody><tr><td>Chat / domain assistant</td><td>A canonical prompt string, chosen once and frozen</td></tr><tr><td>Tool-heavy</td><td>Run <code>agent-sdk call &lt;tool&gt;</code> first to pin what the tool returns, then freeze the prompt that triggers it</td></tr><tr><td>GitHub webhook</td><td><code>agent-sdk github replay &lt;pr&gt; --events &#39;*&#39; --dry-run --out fixtures/github</code> snapshots real payloads for offline replay (<a href="./guides/github.html">GitHub guide</a>)</td></tr><tr><td>PR reviewer with host preparation</td><td>Diff, metadata, and gold labels pinned to commit SHAs; keep any live PR matrix small</td></tr><tr><td>Workspace-dependent</td><td><code>workspaceFiles</code> in <code>t.send</code> options, never developer-machine paths</td></tr></tbody></table><p>Tag the fast, reliably passing core <code>smoke</code> and run <code>--tag smoke</code> in the inner loop. Leave slow or flaky-prone cases untagged for explicit runs.</p><h3 id="materialize-api-backed-fixtures" tabindex="-1">Materialize API-backed fixtures <a class="header-anchor" href="#materialize-api-backed-fixtures" aria-label="Permalink to &quot;Materialize API-backed fixtures&quot;">​</a></h3><p>An input that only points at external data, such as a pull request URL, snapshot id, or pair of commit SHAs, is not self-contained. Fetch it once and commit the rendered fixture before you expand the suite.</p><ol><li>Save the diff, metadata, and labels under <code>fixtures/</code> at pinned revisions.</li><li>Seed those files with <code>workspaceFiles</code>, or read them from the fixture directory.</li><li>Assert decisions and output shape against the saved evidence.</li><li>Keep a small <code>smoke</code> subset for any remaining live pipeline checks.</li></ol><p>Read committed fixtures with <code>@cursor/july/evals/loaders</code>: <code>loadJson</code>, <code>loadJsonl</code>, and <code>loadYaml</code> resolve relative paths against the project root the runner discovered, not the cwd the CLI was invoked from (<code>resolveFixturePath</code> and <code>evalFixtureRoot</code> expose the same resolution for other file formats).</p><p><code>maxConcurrency</code> limits parallel datapoints. It does not limit model or API fan-out inside one datapoint. Materialized fixtures prevent a large suite from exhausting provider and GitHub rate limits. The <a href="./../skills/evals/SKILL.html">evals skill</a> has the full fixture workflow.</p><h2 id="keep-improvements-with-regression-evals" tabindex="-1">Keep improvements with regression evals <a class="header-anchor" href="#keep-improvements-with-regression-evals" aria-label="Permalink to &quot;Keep improvements with regression evals&quot;">​</a></h2><p>Every <a href="./hillclimbing.html">hillclimb</a> round that keeps a change must land an eval that would have failed before the change. If you can&#39;t express the improvement as a gate (a <code>calledTool</code> shift, a bounded <code>action.result</code> count, an output-shape regex), the improvement is unverified, and it&#39;ll regress silently.</p><p>The rule cuts the other way too: never weaken an existing gate to make a round pass. That&#39;s the freeze line moving, and it turns your regression suite into a list of checks that no longer protect anything.</p><h2 id="compare-variants-on-live-traffic" tabindex="-1">Compare variants on live traffic <a class="header-anchor" href="#compare-variants-on-live-traffic" aria-label="Permalink to &quot;Compare variants on live traffic&quot;">​</a></h2><p>Use <code>defineAB</code> to compare variant metrics on live sessions. It is not a test runner and has no <code>agent-sdk ab</code> command. Keep <code>defineEval</code> as the regression ratchet. Eval sessions do not enroll or change live metrics. See <a href="./ab.html">Live A/B metrics</a> for assignment, behavior, collection, and inspection.</p><h2 id="what-s-next" tabindex="-1">What&#39;s next <a class="header-anchor" href="#what-s-next" aria-label="Permalink to &quot;What&#39;s next&quot;">​</a></h2><p>Continue with these pages:</p><ul><li><a href="./ab.html">Live A/B metrics</a>: sticky variants and cumulative metrics on live sessions</li><li><a href="./hillclimbing.html">Hillclimbing</a>: the loop evals make trustworthy</li><li><a href="./building-with-agents.html">Building agents with agents</a>: have a coding agent write the first suite</li><li><a href="./guides/github.html">GitHub guide</a>: deterministic webhook fixtures with <code>github replay</code></li><li><a href="./reference/sessions.html">Sessions and streaming</a>: the events <code>t.events</code> contains</li></ul>`,85)])])}const g=e(n,[["render",l]]);export{k as __pageData,g as default};
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> status</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> evalrun_…</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span></span></code></pre></div><h2 id="what-good-cases-assert" tabindex="-1">What good cases assert <a class="header-anchor" href="#what-good-cases-assert" aria-label="Permalink to &quot;What good cases assert&quot;">​</a></h2><p>Gate decisions and shape, not prose. Model wording varies run to run. Tool choice, tool avoidance, and output structure are the stable contract.</p><ol><li><code>t.succeeded()</code>: always, first.</li><li>The tool decision: <code>calledTool</code> for the intended path, <code>notCalledTool</code> for the likely wrong alternative. The pair is stronger than either alone.</li><li>Output shape: a regex for the contract (<code>/ready|blocked/i</code>, a JSON marker, a findings-block fence), never exact sentences.</li><li>For structured output, parse <code>t.reply</code> and check fields with <code>satisfies</code> instead of substring-matching JSON.</li></ol><p>The common failure modes: asserting exact phrasing, packing more than about five gates into one case (split it), and cases that depend on live external state that drifts (pin the input; see fixtures).</p><h2 id="pick-fixtures-by-agent-type" tabindex="-1">Pick fixtures by agent type <a class="header-anchor" href="#pick-fixtures-by-agent-type" aria-label="Permalink to &quot;Pick fixtures by agent type&quot;">​</a></h2><p>The right fixture depends on the surface under test.</p><table tabindex="0"><thead><tr><th>Agent surface</th><th>Fixture</th></tr></thead><tbody><tr><td>Chat / domain assistant</td><td>A canonical prompt string, chosen once and frozen</td></tr><tr><td>Tool-heavy</td><td>Run <code>agent-sdk call &lt;tool&gt;</code> first to pin what the tool returns, then freeze the prompt that triggers it</td></tr><tr><td>GitHub webhook</td><td><code>agent-sdk github replay &lt;pr&gt; --events &#39;*&#39; --dry-run --out fixtures/github</code> snapshots real payloads for offline replay (<a href="./guides/github.html">GitHub guide</a>)</td></tr><tr><td>PR reviewer with host preparation</td><td>Diff, metadata, and gold labels pinned to commit SHAs; keep any live PR matrix small</td></tr><tr><td>Workspace-dependent</td><td><code>workspaceFiles</code> in <code>t.send</code> options, never developer-machine paths</td></tr></tbody></table><p>Tag the fast, reliably passing core <code>smoke</code> and run <code>--tag smoke</code> in the inner loop. Leave slow or flaky-prone cases untagged for explicit runs.</p><h3 id="materialize-api-backed-fixtures" tabindex="-1">Materialize API-backed fixtures <a class="header-anchor" href="#materialize-api-backed-fixtures" aria-label="Permalink to &quot;Materialize API-backed fixtures&quot;">​</a></h3><p>An input that only points at external data, such as a pull request URL, snapshot id, or pair of commit SHAs, is not self-contained. Fetch it once and commit the rendered fixture before you expand the suite.</p><ol><li>Save the diff, metadata, and labels under <code>fixtures/</code> at pinned revisions.</li><li>Seed those files with <code>workspaceFiles</code>, or read them from the fixture directory.</li><li>Assert decisions and output shape against the saved evidence.</li><li>Keep a small <code>smoke</code> subset for any remaining live pipeline checks.</li></ol><p>Read committed fixtures with <code>@cursor/july/evals/loaders</code>: <code>loadJson</code>, <code>loadJsonl</code>, and <code>loadYaml</code> resolve relative paths against the project root the runner discovered, not the cwd the CLI was invoked from (<code>resolveFixturePath</code> and <code>evalFixtureRoot</code> expose the same resolution for other file formats).</p><p><code>maxConcurrency</code> limits parallel datapoints. It does not limit model or API fan-out inside one datapoint. Materialized fixtures prevent a large suite from exhausting provider and GitHub rate limits. The <a href="./../skills/evals/SKILL.html">evals skill</a> has the full fixture workflow.</p><h2 id="keep-improvements-with-regression-evals" tabindex="-1">Keep improvements with regression evals <a class="header-anchor" href="#keep-improvements-with-regression-evals" aria-label="Permalink to &quot;Keep improvements with regression evals&quot;">​</a></h2><p>Every <a href="./hillclimbing.html">hillclimb</a> round that keeps a change must land an eval that would have failed before the change. If you can&#39;t express the improvement as a gate (a <code>calledTool</code> shift, a bounded <code>action.result</code> count, an output-shape regex), the improvement is unverified, and it&#39;ll regress silently.</p><p>The rule cuts the other way too: never weaken an existing gate to make a round pass. That&#39;s the freeze line moving, and it turns your regression suite into a list of checks that no longer protect anything.</p><h2 id="compare-variants-on-live-traffic" tabindex="-1">Compare variants on live traffic <a class="header-anchor" href="#compare-variants-on-live-traffic" aria-label="Permalink to &quot;Compare variants on live traffic&quot;">​</a></h2><p>Use <code>defineAB</code> to compare variant metrics on live sessions. It is not a test runner and has no <code>agent-sdk ab</code> command. Keep <code>defineEval</code> as the regression ratchet. Eval sessions do not enroll or change live metrics. See <a href="./ab.html">Live A/B metrics</a> for assignment, behavior, collection, and inspection.</p><h2 id="what-s-next" tabindex="-1">What&#39;s next <a class="header-anchor" href="#what-s-next" aria-label="Permalink to &quot;What&#39;s next&quot;">​</a></h2><p>Continue with these pages:</p><ul><li><a href="./ab.html">Live A/B metrics</a>: sticky variants and cumulative metrics on live sessions</li><li><a href="./hillclimbing.html">Hillclimbing</a>: the loop evals make trustworthy</li><li><a href="./building-with-agents.html">Building agents with agents</a>: have a coding agent write the first suite</li><li><a href="./guides/github.html">GitHub guide</a>: deterministic webhook fixtures with <code>github replay</code></li><li><a href="./reference/sessions.html">Sessions and streaming</a>: the events <code>t.events</code> contains</li></ul>`,82)])])}const g=e(n,[["render",l]]);export{k as __pageData,g as default};

package/dist/docs/assets/{evals.md.a0SMN6r9.lean.js → evals.md.lfJoEVc8.lean.js}

@@ -1 +1 @@
-import{_ as e,c as a,o as i,ag as t}from"./chunks/framework.BCISBCiQ.js";const k=JSON.parse('{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agent-sdk eval, and use them as regression checks.","frontmatter":{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agent-sdk eval, and use them as regression checks."},"headers":[],"relativePath":"evals.md","filePath":"evals.md"}'),n={name:"evals.md"};function l(d,s,o,h,r,p){return i(),a("div",null,[...s[0]||(s[0]=[t("",85)])])}const g=e(n,[["render",l]]);export{k as __pageData,g as default};
+import{_ as e,c as i,o as a,ag as t}from"./chunks/framework.BCISBCiQ.js";const k=JSON.parse('{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agent-sdk eval, and use them as regression checks.","frontmatter":{"title":"Evals","description":"Define repeatable checks with defineEval, run them with agent-sdk eval, and use them as regression checks."},"headers":[],"relativePath":"evals.md","filePath":"evals.md"}'),n={name:"evals.md"};function l(d,s,h,o,r,p){return a(),i("div",null,[...s[0]||(s[0]=[t("",82)])])}const g=e(n,[["render",l]]);export{k as __pageData,g as default};

package/dist/docs/assets/{example-agents_approval-buddy.md.DNL83puR.js → example-agents_approval-buddy.md.DmezILPg.js}

@@ -1,4 +1,4 @@
-import{_ as a,c as t,o as s,ag as i}from"./chunks/framework.BCISBCiQ.js";const u=JSON.parse('{"title":"Keep PR approval policy deterministic with Approval Buddy","description":"Separate code-owned eligibility from model-owned review, then connect GitHub, Slack, subagents, durable storage, and evals.","frontmatter":{"title":"Keep PR approval policy deterministic with Approval Buddy","description":"Separate code-owned eligibility from model-owned review, then connect GitHub, Slack, subagents, durable storage, and evals."},"headers":[],"relativePath":"example-agents/approval-buddy.md","filePath":"example-agents/approval-buddy.md"}'),o={name:"example-agents/approval-buddy.md"};function r(l,e,n,d,p,h){return s(),t("div",null,[...e[0]||(e[0]=[i(`<h1 id="keep-pr-approval-policy-deterministic-with-approval-buddy" tabindex="-1">Keep PR approval policy deterministic with Approval Buddy <a class="header-anchor" href="#keep-pr-approval-policy-deterministic-with-approval-buddy" aria-label="Permalink to &quot;Keep PR approval policy deterministic with Approval Buddy&quot;">​</a></h1><p>Approval Buddy approves eligible pull requests from a fixed roster and declines every other request. GitHub still blocks self-approval when the stamp identity authored the PR. Code decides eligibility. The model prepares evidence, runs two specialist reviews, and passes their findings to the approval tool without changing the policy decision.</p><p>Use this example when an agent can make a judgment inside a workflow, but authorization and the final side effect must stay in deterministic code.</p><p><a href="./../../examples/approval-buddy/">Browse the Approval Buddy source.</a></p><h2 id="keep-approval-policy-in-code" tabindex="-1">Keep approval policy in code <a class="header-anchor" href="#keep-approval-policy-in-code" aria-label="Permalink to &quot;Keep approval policy in code&quot;">​</a></h2><p>Approval Buddy draws three hard boundaries:</p><ul><li><code>prepare_review</code> and <code>approve_pr</code> re-read the live PR and apply the same eligibility rules.</li><li>Two subagents inspect prepared evidence, but their findings never grant or block approval.</li><li>Only <code>approve_pr</code> posts the GitHub review.</li></ul><p>A spoofed webhook, Slack message, or model claim can&#39;t add someone to the buddy roster. The mutating tool checks the source of truth immediately before it acts.</p><h2 id="follow-the-intended-stamp-flow" tabindex="-1">Follow the intended stamp flow <a class="header-anchor" href="#follow-the-intended-stamp-flow" aria-label="Permalink to &quot;Follow the intended stamp flow&quot;">​</a></h2><p>The root instructions ask the model to run this sequence for a qualifying PR:</p><ol><li>A non-draft <code>pull_request</code> event arrives with action <code>opened</code>, <code>reopened</code>, or <code>ready_for_review</code>.</li><li>The GitHub channel checks its repository allowlist and starts a session.</li><li><code>turn.started</code> posts a pending commit status.</li><li>The model calls <code>prepare_review</code>.</li><li>Host code fetches the live PR. It checks the author, open state, merged state, and draft state.</li><li>A qualifying PR gets <code>pr/MANIFEST.md</code>, <code>pr/meta.json</code>, and <code>pr/diff.patch</code> in the session workspace. Diffs above 2,000,000 characters are truncated and marked in metadata.</li><li>The model calls both review subagents through the built-in <code>task</code> tool.</li><li>It concatenates their contracted replies and calls <code>approve_pr</code>.</li><li><code>approve_pr</code> re-runs eligibility, posts an <code>APPROVE</code> review, and returns the outcome.</li><li>The channel posts a final commit status. A self-approval block also gets a short timeline comment because no approval review can appear.</li></ol><p>Ineligible PRs skip evidence and subagents. The model still calls <code>approve_pr</code> so the deterministic tool returns the formal decline reason.</p><p>Steps 4 through 9 are prompt-driven. The channel doesn&#39;t enforce tool order or prove both subagents ran, and <code>approve_pr</code> accepts missing findings. A failed turn clears the pending status with a green non-blocking result without approving the PR.</p><h2 id="map-the-framework-features" tabindex="-1">Map the framework features <a class="header-anchor" href="#map-the-framework-features" aria-label="Permalink to &quot;Map the framework features&quot;">​</a></h2><table tabindex="0"><thead><tr><th>Capability</th><th>Source</th><th>Role</th></tr></thead><tbody><tr><td>Root agent and policy prompt</td><td><a href="../../examples/approval-buddy/agent/agent.ts"><code>agent/agent.ts</code></a>, <a href="./../../examples/approval-buddy/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Configure the local agent and describe orchestration order.</td></tr><tr><td>GitHub channel</td><td><a href="../../examples/approval-buddy/agent/channels/github.ts"><code>agent/channels/github.ts</code></a></td><td>Filter wakes, lease GitHub access, and publish status events.</td></tr><tr><td>Slack channel</td><td><a href="../../examples/approval-buddy/agent/channels/slack.ts"><code>agent/channels/slack.ts</code></a></td><td>Accept approval-bot stamp and qualification requests.</td></tr><tr><td>Server tools</td><td><a href="./../../examples/approval-buddy/agent/tools/"><code>agent/tools/</code></a></td><td>Prepare evidence, approve, list buddies, and search GIFs.</td></tr><tr><td>Deterministic policy</td><td><a href="../../examples/approval-buddy/agent/lib/approve.ts"><code>agent/lib/approve.ts</code></a>, <a href="../../examples/approval-buddy/agent/lib/buddies.ts"><code>agent/lib/buddies.ts</code></a></td><td>Own the roster and live eligibility checks.</td></tr><tr><td>Review subagents</td><td><a href="./../../examples/approval-buddy/agent/subagents/"><code>agent/subagents/</code></a></td><td>Run deep audit and code-quality passes over the same evidence.</td></tr><tr><td>Storage</td><td><a href="../../examples/approval-buddy/agent/storage.ts"><code>agent/storage.ts</code></a></td><td>Persist sessions and events with <code>cursorHostedStorage</code> (Bugbot <code>agent_serve_*</code>).</td></tr><tr><td>Live A/B experiment</td><td><a href="../../examples/approval-buddy/agent/ab.ts"><code>agent/ab.ts</code></a></td><td>Compare baseline responses with a concise, presentation-only treatment (<code>concise-results</code>).</td></tr><tr><td>Evals and unit tests</td><td><a href="./../../examples/approval-buddy/evals/"><code>evals/</code></a>, <a href="./../../examples/approval-buddy/agent/lib/"><code>agent/lib/</code></a></td><td>Protect routing, output contracts, policy, and GitHub behavior.</td></tr></tbody></table><p>There are no authored skills, MCP connections, schedules, reminders, hooks, sandbox seeds, or tool approvals.</p><h2 id="prepare-credentials" tabindex="-1">Prepare credentials <a class="header-anchor" href="#prepare-credentials" aria-label="Permalink to &quot;Prepare credentials&quot;">​</a></h2><p>You need:</p><ul><li>Node 22.13 or newer.</li><li>An agent-runtime credential.</li><li>GitHub access to read PRs, post reviews, create commit statuses, and post the self-approval visibility comment.</li></ul><p>Optional GIF selection uses:</p><ul><li><code>GIPHY_API_KEY</code> or <code>APPROVAL_BUDDY_GIPHY_API_KEY</code>,</li><li><code>APPROVAL_BUDDY_STAMP_GIF</code>, or</li><li>severity-specific <code>APPROVAL_BUDDY_STAMP_GIF_&lt;LEVEL&gt;</code> variables.</li></ul><p>If you enable Giphy in a hosted copy, declare its secret and <code>api.giphy.com</code> egress.</p><h2 id="validate-without-approving-a-pr" tabindex="-1">Validate without approving a PR <a class="header-anchor" href="#validate-without-approving-a-pr" aria-label="Permalink to &quot;Validate without approving a PR&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span></span>
+import{_ as a,c as t,o as s,ag as i}from"./chunks/framework.BCISBCiQ.js";const u=JSON.parse('{"title":"Keep PR approval policy deterministic with Approval Buddy","description":"Separate code-owned eligibility from model-owned review, then connect GitHub, Slack, subagents, durable storage, and evals.","frontmatter":{"title":"Keep PR approval policy deterministic with Approval Buddy","description":"Separate code-owned eligibility from model-owned review, then connect GitHub, Slack, subagents, durable storage, and evals."},"headers":[],"relativePath":"example-agents/approval-buddy.md","filePath":"example-agents/approval-buddy.md"}'),o={name:"example-agents/approval-buddy.md"};function r(l,e,n,d,p,h){return s(),t("div",null,[...e[0]||(e[0]=[i(`<h1 id="keep-pr-approval-policy-deterministic-with-approval-buddy" tabindex="-1">Keep PR approval policy deterministic with Approval Buddy <a class="header-anchor" href="#keep-pr-approval-policy-deterministic-with-approval-buddy" aria-label="Permalink to &quot;Keep PR approval policy deterministic with Approval Buddy&quot;">​</a></h1><p>Approval Buddy approves eligible pull requests from a fixed roster and declines every other request. GitHub still blocks self-approval when the stamp identity authored the PR. Code decides eligibility. The model prepares evidence, runs two specialist reviews, and passes their findings to the approval tool without changing the policy decision.</p><p>Use this example when an agent can make a judgment inside a workflow, but authorization and the final side effect must stay in deterministic code.</p><p><a href="./../../examples/approval-buddy/">Browse the Approval Buddy source.</a></p><h2 id="keep-approval-policy-in-code" tabindex="-1">Keep approval policy in code <a class="header-anchor" href="#keep-approval-policy-in-code" aria-label="Permalink to &quot;Keep approval policy in code&quot;">​</a></h2><p>Approval Buddy draws three hard boundaries:</p><ul><li><code>prepare_review</code> and <code>approve_pr</code> re-read the live PR and apply the same eligibility rules.</li><li>Two subagents inspect prepared evidence, but their findings never grant or block approval.</li><li>Only <code>approve_pr</code> posts the GitHub review.</li></ul><p>A spoofed webhook, Slack message, or model claim can&#39;t add someone to the buddy roster. The mutating tool checks the source of truth immediately before it acts.</p><h2 id="follow-the-intended-stamp-flow" tabindex="-1">Follow the intended stamp flow <a class="header-anchor" href="#follow-the-intended-stamp-flow" aria-label="Permalink to &quot;Follow the intended stamp flow&quot;">​</a></h2><p>The root instructions ask the model to run this sequence for a qualifying PR:</p><ol><li>A non-draft <code>pull_request</code> event arrives with action <code>opened</code>, <code>reopened</code>, or <code>ready_for_review</code>.</li><li>The GitHub channel checks its repository allowlist and starts a session.</li><li><code>turn.started</code> posts a pending commit status.</li><li>The model calls <code>prepare_review</code>.</li><li>Host code fetches the live PR. It checks the author, open state, merged state, and draft state.</li><li>A qualifying PR gets <code>pr/MANIFEST.md</code>, <code>pr/meta.json</code>, and <code>pr/diff.patch</code> in the session workspace. Diffs above 2,000,000 characters are truncated and marked in metadata.</li><li>The model calls both review subagents through the built-in <code>task</code> tool.</li><li>It concatenates their contracted replies and calls <code>approve_pr</code>.</li><li><code>approve_pr</code> re-runs eligibility, posts an <code>APPROVE</code> review, and returns the outcome.</li><li>The channel posts a final commit status. A self-approval block also gets a short timeline comment because no approval review can appear.</li></ol><p>Ineligible PRs skip evidence and subagents. The model still calls <code>approve_pr</code> so the deterministic tool returns the formal decline reason.</p><p>Steps 4 through 9 are prompt-driven. The channel doesn&#39;t enforce tool order or prove both subagents ran, and <code>approve_pr</code> accepts missing findings. A failed turn clears the pending status with a green non-blocking result without approving the PR.</p><h2 id="map-the-framework-features" tabindex="-1">Map the framework features <a class="header-anchor" href="#map-the-framework-features" aria-label="Permalink to &quot;Map the framework features&quot;">​</a></h2><table tabindex="0"><thead><tr><th>Capability</th><th>Source</th><th>Role</th></tr></thead><tbody><tr><td>Root agent and policy prompt</td><td><a href="../../examples/approval-buddy/agent/agent.ts"><code>agent/agent.ts</code></a>, <a href="./../../examples/approval-buddy/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Configure the local agent and describe orchestration order.</td></tr><tr><td>GitHub channel</td><td><a href="../../examples/approval-buddy/agent/channels/github.ts"><code>agent/channels/github.ts</code></a></td><td>Filter wakes, lease GitHub access, and publish status events.</td></tr><tr><td>Slack channel</td><td><a href="../../examples/approval-buddy/agent/channels/slack.ts"><code>agent/channels/slack.ts</code></a></td><td>Accept approval-bot stamp and qualification requests.</td></tr><tr><td>Server tools</td><td><a href="./../../examples/approval-buddy/agent/tools/"><code>agent/tools/</code></a></td><td>Prepare evidence, approve, list buddies, and search GIFs.</td></tr><tr><td>Deterministic policy</td><td><a href="../../examples/approval-buddy/agent/lib/approve.ts"><code>agent/lib/approve.ts</code></a>, <a href="../../examples/approval-buddy/agent/lib/buddies.ts"><code>agent/lib/buddies.ts</code></a></td><td>Own the roster and live eligibility checks.</td></tr><tr><td>Review subagents</td><td><a href="./../../examples/approval-buddy/agent/subagents/"><code>agent/subagents/</code></a></td><td>Run deep audit and code-quality passes over the same evidence.</td></tr><tr><td>Storage</td><td><a href="../../examples/approval-buddy/agent/storage.ts"><code>agent/storage.ts</code></a></td><td>Persist sessions and events with <code>cursorHostedStorage</code>. See <a href="./../storage.html">Storage</a>.</td></tr><tr><td>Live A/B experiment</td><td><a href="../../examples/approval-buddy/agent/ab.ts"><code>agent/ab.ts</code></a></td><td>Compare baseline responses with a concise, presentation-only treatment (<code>concise-results</code>).</td></tr><tr><td>Evals and unit tests</td><td><a href="./../../examples/approval-buddy/evals/"><code>evals/</code></a>, <a href="./../../examples/approval-buddy/agent/lib/"><code>agent/lib/</code></a></td><td>Protect routing, output contracts, policy, and GitHub behavior.</td></tr></tbody></table><p>There are no authored skills, MCP connections, schedules, reminders, hooks, sandbox seeds, or tool approvals.</p><h2 id="prepare-credentials" tabindex="-1">Prepare credentials <a class="header-anchor" href="#prepare-credentials" aria-label="Permalink to &quot;Prepare credentials&quot;">​</a></h2><p>You need:</p><ul><li>Node 22.13 or newer.</li><li>An agent-runtime credential.</li><li>GitHub access to read PRs, post reviews, create commit statuses, and post the self-approval visibility comment.</li></ul><p>Optional GIF selection uses:</p><ul><li><code>GIPHY_API_KEY</code> or <code>APPROVAL_BUDDY_GIPHY_API_KEY</code>,</li><li><code>APPROVAL_BUDDY_STAMP_GIF</code>, or</li><li>severity-specific <code>APPROVAL_BUDDY_STAMP_GIF_&lt;LEVEL&gt;</code> variables.</li></ul><p>If you enable Giphy in a hosted copy, declare its secret and <code>api.giphy.com</code> egress.</p><h2 id="validate-without-approving-a-pr" tabindex="-1">Validate without approving a PR <a class="header-anchor" href="#validate-without-approving-a-pr" aria-label="Permalink to &quot;Validate without approving a PR&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> info</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p>List the deterministic roster:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> call</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> list_buddies</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/approval-buddy</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --input</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &#39;{}&#39;</span></span></code></pre></div><p>Set a known merged PR, then run the read-only precheck:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">MERGED_PR_URL</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">https://github.com/your-org/your-repo/pull/123</span></span>

package/dist/docs/assets/{example-agents_benny.md.C40vHRLc.js → example-agents_benny.md.B0kwY7D_.js}

@@ -1,7 +1,5 @@
-import{_ as a,c as t,o as s,ag as i}from"./chunks/framework.BCISBCiQ.js";const k=JSON.parse('{"title":"Route Slack work through repository playbooks","description":"Combine account-linked chat, allowlisted Socket Mode channel watching, inherited repository skills, and a custom local workspace.","frontmatter":{"title":"Route Slack work through repository playbooks","description":"Combine account-linked chat, allowlisted Socket Mode channel watching, inherited repository skills, and a custom local workspace."},"headers":[],"relativePath":"example-agents/benny.md","filePath":"example-agents/benny.md"}'),o={name:"example-agents/benny.md"};function n(l,e,r,h,d,p){return s(),t("div",null,[...e[0]||(e[0]=[i(`<h1 id="route-slack-work-through-repository-playbooks" tabindex="-1">Route Slack work through repository playbooks <a class="header-anchor" href="#route-slack-work-through-repository-playbooks" aria-label="Permalink to &quot;Route Slack work through repository playbooks&quot;">​</a></h1><p>This agent is a Slack teammate for a product team. Mentions and direct messages reach it through an account-linked transport. New top-level posts in an allowlisted issue channel reach it through a dedicated Slack app, even without a mention. The agent then selects a repository playbook for triage, reproduction, fixes, reviews, on-call work, or design critique.</p><p>Use this example when Slack is the intake surface and your durable procedures already live as repository skills.</p><p><a href="./../../examples/benny/">Browse the current playbook-router source.</a></p><h2 id="combine-two-slack-transports-with-repo-skills" tabindex="-1">Combine two Slack transports with repo skills <a class="header-anchor" href="#combine-two-slack-transports-with-repo-skills" aria-label="Permalink to &quot;Combine two Slack transports with repo skills&quot;">​</a></h2><p>The playbook router uniquely combines three decisions:</p><ul><li>Two Slack transports serve different engagement modes.</li><li><code>local.cwd</code> keeps session workspaces inside the monorepo.</li><li>Instructions route work to inherited repository playbooks instead of authored <code>agent/skills/</code>.</li></ul><p>The result is a thin agent project over a mature procedure library.</p><h2 id="follow-an-issue-report" tabindex="-1">Follow an issue report <a class="header-anchor" href="#follow-an-issue-report" aria-label="Permalink to &quot;Follow an issue report&quot;">​</a></h2><ol><li>A teammate creates a top-level post in the allowlisted issue channel.</li><li>The dedicated Socket Mode channel accepts the allowlisted channel.</li><li>A 15-second debounce lets edits settle. Deleting the post during that window cancels the dispatch.</li><li>The Agent SDK creates a thread-scoped session and sends the report to the playbook router.</li><li>The instructions select the matching triage playbook.</li><li>The harness finds the repository root, opens the inherited playbook, and follows its procedure.</li><li>The agent posts only in the source thread and reports the evidence it gathered.</li></ol><p>Mentions and direct messages follow the same agent instructions. They don&#39;t need the watched-channel path.</p><h2 id="map-the-playbook-router-files" tabindex="-1">Map the playbook router files <a class="header-anchor" href="#map-the-playbook-router-files" aria-label="Permalink to &quot;Map the playbook router files&quot;">​</a></h2><table tabindex="0"><thead><tr><th>File</th><th>Purpose</th></tr></thead><tbody><tr><td><a href="../../examples/benny/agent/agent.ts"><code>agent/agent.ts</code></a></td><td>Names the agent, selects its model, and keeps the harness under <code>.agent-serve/harness</code>.</td></tr><tr><td><a href="./../../examples/benny/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Defines engagement rules, evidence policy, and the playbook routing map.</td></tr><tr><td><a href="../../examples/benny/agent/channels/slack.ts"><code>agent/channels/slack.ts</code></a></td><td>Handles account-linked mentions and direct messages.</td></tr><tr><td><a href="../../examples/benny/agent/channels/slack-app.ts"><code>agent/channels/slack-app.ts</code></a></td><td>Runs the dedicated app and watches one allowlisted channel.</td></tr><tr><td><a href="../../examples/benny/agent/storage.ts"><code>agent/storage.ts</code></a></td><td>Persists sessions and events with <code>cursorHostedStorage</code>.</td></tr><tr><td><a href="../../examples/benny/evals/evals.config.ts"><code>evals/evals.config.ts</code></a></td><td>Caps eval run concurrency.</td></tr><tr><td><a href="../../examples/benny/evals/smoke.eval.ts"><code>evals/smoke.eval.ts</code></a></td><td>Checks the agent identity and expected triage route.</td></tr></tbody></table><p>The playbook router authors no tools, MCP connections, subagents, schedules, hooks, A/B experiments, or sandbox seeds.</p><h2 id="see-why-local-cwd-matters" tabindex="-1">See why <code>local.cwd</code> matters <a class="header-anchor" href="#see-why-local-cwd-matters" aria-label="Permalink to &quot;See why \`local.cwd\` matters&quot;">​</a></h2><p>The Agent SDK normally keeps an ephemeral <code>run</code> or <code>eval</code> workspace outside a large monorepo. This prevents ancestor instruction and repository-rule files from leaking into an unrelated agent.</p><p>The playbook router needs the opposite. Its procedures live at the repository root, so <code>agent.ts</code> sets:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">local</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">  cwd</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;.agent-serve/harness&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Each harness workspace lands under <code>examples/benny/.agent-serve/harness/&lt;sessionId&gt;</code>. Walking up the directory tree reaches the host repository and its inherited playbook directory.</p><p>Those playbooks are inherited context. <code>agent-sdk info</code> reports zero authored skills for the agent. Copying this project into another repository removes its main procedures unless you copy or replace the skill library too.</p><h2 id="connect-both-slack-paths" tabindex="-1">Connect both Slack paths <a class="header-anchor" href="#connect-both-slack-paths" aria-label="Permalink to &quot;Connect both Slack paths&quot;">​</a></h2><p>The account-linked path needs an agent-runtime login and a connected Slack account:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> login</span></span>
+import{_ as a,c as t,o as s,ag as o}from"./chunks/framework.BCISBCiQ.js";const k=JSON.parse('{"title":"Route Slack work through repository playbooks","description":"Combine account-linked chat, allowlisted Socket Mode channel watching, inherited repository skills, and a custom local workspace.","frontmatter":{"title":"Route Slack work through repository playbooks","description":"Combine account-linked chat, allowlisted Socket Mode channel watching, inherited repository skills, and a custom local workspace."},"headers":[],"relativePath":"example-agents/benny.md","filePath":"example-agents/benny.md"}'),i={name:"example-agents/benny.md"};function n(l,e,r,h,d,c){return s(),t("div",null,[...e[0]||(e[0]=[o(`<h1 id="route-slack-work-through-repository-playbooks" tabindex="-1">Route Slack work through repository playbooks <a class="header-anchor" href="#route-slack-work-through-repository-playbooks" aria-label="Permalink to &quot;Route Slack work through repository playbooks&quot;">​</a></h1><p>This agent is a Slack teammate for a product team. Mentions and direct messages reach it through an account-linked transport. New top-level posts in an allowlisted issue channel reach it through a dedicated Slack app, even without a mention. The agent then selects a repository playbook for triage, reproduction, fixes, reviews, on-call work, or design critique.</p><p>Use this example when Slack is the intake surface and your durable procedures already live as repository skills.</p><p><a href="./../../examples/benny/">Browse the current playbook-router source.</a></p><h2 id="combine-two-slack-transports-with-repo-skills" tabindex="-1">Combine two Slack transports with repo skills <a class="header-anchor" href="#combine-two-slack-transports-with-repo-skills" aria-label="Permalink to &quot;Combine two Slack transports with repo skills&quot;">​</a></h2><p>The playbook router uniquely combines three decisions:</p><ul><li>Two Slack transports serve different engagement modes.</li><li><code>local.cwd</code> keeps session workspaces inside the monorepo.</li><li>Instructions route work to inherited repository playbooks instead of authored <code>agent/skills/</code>.</li></ul><p>The result is a thin agent project over a mature procedure library.</p><h2 id="follow-an-issue-report" tabindex="-1">Follow an issue report <a class="header-anchor" href="#follow-an-issue-report" aria-label="Permalink to &quot;Follow an issue report&quot;">​</a></h2><ol><li>A teammate creates a top-level post in the allowlisted issue channel.</li><li>The dedicated Socket Mode channel accepts the allowlisted channel.</li><li>A 15-second debounce lets edits settle. Deleting the post during that window cancels the dispatch.</li><li>The Agent SDK creates a thread-scoped session and sends the report to the playbook router.</li><li>The instructions select the matching triage playbook.</li><li>The harness finds the repository root, opens the inherited playbook, and follows its procedure.</li><li>The agent posts only in the source thread and reports the evidence it gathered.</li></ol><p>Mentions and direct messages follow the same agent instructions. They don&#39;t need the watched-channel path.</p><h2 id="map-the-playbook-router-files" tabindex="-1">Map the playbook router files <a class="header-anchor" href="#map-the-playbook-router-files" aria-label="Permalink to &quot;Map the playbook router files&quot;">​</a></h2><table tabindex="0"><thead><tr><th>File</th><th>Purpose</th></tr></thead><tbody><tr><td><a href="../../examples/benny/agent/agent.ts"><code>agent/agent.ts</code></a></td><td>Names the agent, selects its model, and points the harness at a project-local cwd so inherited playbooks load.</td></tr><tr><td><a href="./../../examples/benny/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Defines engagement rules, evidence policy, and the playbook routing map.</td></tr><tr><td><a href="../../examples/benny/agent/channels/slack.ts"><code>agent/channels/slack.ts</code></a></td><td>Handles account-linked mentions and direct messages.</td></tr><tr><td><a href="../../examples/benny/agent/channels/slack-app.ts"><code>agent/channels/slack-app.ts</code></a></td><td>Runs the dedicated app and watches one allowlisted channel.</td></tr><tr><td><a href="../../examples/benny/agent/storage.ts"><code>agent/storage.ts</code></a></td><td>Persists sessions and events with <code>cursorHostedStorage</code>.</td></tr><tr><td><a href="../../examples/benny/evals/evals.config.ts"><code>evals/evals.config.ts</code></a></td><td>Caps eval run concurrency.</td></tr><tr><td><a href="../../examples/benny/evals/smoke.eval.ts"><code>evals/smoke.eval.ts</code></a></td><td>Checks the agent identity and expected triage route.</td></tr></tbody></table><p>The playbook router authors no tools, MCP connections, subagents, schedules, hooks, A/B experiments, or sandbox seeds.</p><h2 id="see-why-local-cwd-matters" tabindex="-1">See why <code>local.cwd</code> matters <a class="header-anchor" href="#see-why-local-cwd-matters" aria-label="Permalink to &quot;See why \`local.cwd\` matters&quot;">​</a></h2><p>The Agent SDK normally keeps an ephemeral <code>run</code> or <code>eval</code> workspace outside a large monorepo. This prevents ancestor instruction and repository-rule files from leaking into an unrelated agent.</p><p>The playbook router needs the opposite. Its procedures live at the repository root, so <code>agent.ts</code> points <code>local.cwd</code> at a harness directory under the project. Each harness workspace is a child of that directory. Walking up reaches the host repository and its inherited playbook directory.</p><p>Those playbooks are inherited context. <code>agent-sdk info</code> reports zero authored skills for the agent. Copying this project into another repository removes its main procedures unless you copy or replace the skill library too.</p><h2 id="connect-both-slack-paths" tabindex="-1">Connect both Slack paths <a class="header-anchor" href="#connect-both-slack-paths" aria-label="Permalink to &quot;Connect both Slack paths&quot;">​</a></h2><p>The account-linked path needs an agent-runtime login and a connected Slack account:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> login</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> whoami</span></span></code></pre></div><p>It routes explicit mentions without a dedicated Slack token on the host.</p><p>For the watched-channel path, configure a dedicated Socket Mode app with:</p><ul><li>subscribe to <code>message.channels</code> and <code>message.groups</code>,</li><li>have an App-Level Token with <code>connections:write</code>, and</li><li>be a member of the watched channel.</li></ul><p>Run <code>agent-sdk slack create --dir examples/benny --channel-posts</code> for a dedicated Socket Mode app, then <code>agent-sdk slack doctor</code>.</p><p>Missing dedicated-app tokens leave that channel idle. They don&#39;t stop the account-linked channel.</p><h2 id="validate-and-start-the-server" tabindex="-1">Validate and start the server <a class="header-anchor" href="#validate-and-start-the-server" aria-label="Permalink to &quot;Validate and start the server&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/benny</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> info</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/benny</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/benny</span></span></code></pre></div><p>The info output should show two Slack channels and no authored skill. That combination confirms the example is using inherited playbooks.</p><h2 id="exercise-each-engagement-mode" tabindex="-1">Exercise each engagement mode <a class="header-anchor" href="#exercise-each-engagement-mode" aria-label="Permalink to &quot;Exercise each engagement mode&quot;">​</a></h2><p>Test the explicit account-linked path by asking:</p><blockquote><p>Which playbook would you use to triage a product UI bug?</p></blockquote><p>Test the dedicated app:</p><ol><li>Create a top-level post in the allowlisted issue channel.</li><li>Don&#39;t mention the bot.</li><li>Wait for the debounce window.</li><li>Confirm the agent replies in the post&#39;s thread.</li></ol><p>Thread replies don&#39;t trigger the proactive watch. Mentions still use Slack&#39;s normal mention path. Bot-authored posts are ignored to prevent loops.</p><p>The channel uses the default handler after filtering. It doesn&#39;t apply a second code-level classifier, so every accepted top-level post spends a model turn and reaches the prompt.</p><h2 id="inspect-thread-continuity" tabindex="-1">Inspect thread continuity <a class="header-anchor" href="#inspect-thread-continuity" aria-label="Permalink to &quot;Inspect thread continuity&quot;">​</a></h2><p>The Agent SDK keys Slack sessions by channel and thread timestamp. A follow-up in the same thread resumes the conversation and workspace. A new top-level issue gets a new session.</p><p>This lets a playbook gather evidence over several turns without mixing two reports. The playground shows both the account-linked and dedicated-app sessions while the dev server runs.</p><h2 id="run-the-smoke-eval" tabindex="-1">Run the smoke eval <a class="header-anchor" href="#run-the-smoke-eval" aria-label="Permalink to &quot;Run the smoke eval&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/benny</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/benny</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> smoke</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p>The case asks for the agent identity and the playbook used for issue triage. It checks the configured identity and route label.</p><p>This is a lexical smoke test. It doesn&#39;t prove Slack delivery, skill selection, skill loading, procedure execution, or thread-only behavior. Add fixture-backed evals around the playbooks when you reuse this design.</p><h2 id="build-a-playbook-routed-teammate" tabindex="-1">Build a playbook-routed teammate <a class="header-anchor" href="#build-a-playbook-routed-teammate" aria-label="Permalink to &quot;Build a playbook-routed teammate&quot;">​</a></h2><p>Use this structure when your organization already has tested skills:</p><ol><li>Put the playbooks under a stable repository path.</li><li>Set <code>local.cwd</code> so harness workspaces can inherit that path.</li><li>Write a short routing table in <code>instructions.md</code>.</li><li>Use account-linked Slack for explicit requests.</li><li>Add a dedicated app only for allowlisted proactive intake.</li><li>Keep the channel allowlist narrow and debounce edited posts.</li><li>Add an eval for every important request-to-playbook route.</li></ol><p>If the procedures should ship with the agent, put them under <code>agent/skills/</code> instead. Authored skills appear in the manifest and travel with the project.</p><h2 id="where-to-go-next" tabindex="-1">Where to go next <a class="header-anchor" href="#where-to-go-next" aria-label="Permalink to &quot;Where to go next&quot;">​</a></h2><ul><li><a href="./../guides/slack.html">Slack</a></li><li><a href="./../reference/agent-config.html">Agent config</a></li><li><a href="./../reference/skills.html">Skills</a></li><li><a href="./../reference/sessions.html">Sessions and streaming</a></li><li><a href="./../evals.html">Evals</a></li></ul>`,51)])])}const u=a(o,[["render",n]]);export{k as __pageData,u as default};
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/benny</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> smoke</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p>The case asks for the agent identity and the playbook used for issue triage. It checks the configured identity and route label.</p><p>This is a lexical smoke test. It doesn&#39;t prove Slack delivery, skill selection, skill loading, procedure execution, or thread-only behavior. Add fixture-backed evals around the playbooks when you reuse this design.</p><h2 id="build-a-playbook-routed-teammate" tabindex="-1">Build a playbook-routed teammate <a class="header-anchor" href="#build-a-playbook-routed-teammate" aria-label="Permalink to &quot;Build a playbook-routed teammate&quot;">​</a></h2><p>Use this structure when your organization already has tested skills:</p><ol><li>Put the playbooks under a stable repository path.</li><li>Set <code>local.cwd</code> so harness workspaces can inherit that path.</li><li>Write a short routing table in <code>instructions.md</code>.</li><li>Use account-linked Slack for explicit requests.</li><li>Add a dedicated app only for allowlisted proactive intake.</li><li>Keep the channel allowlist narrow and debounce edited posts.</li><li>Add an eval for every important request-to-playbook route.</li></ol><p>If the procedures should ship with the agent, put them under <code>agent/skills/</code> instead. Authored skills appear in the manifest and travel with the project.</p><h2 id="where-to-go-next" tabindex="-1">Where to go next <a class="header-anchor" href="#where-to-go-next" aria-label="Permalink to &quot;Where to go next&quot;">​</a></h2><ul><li><a href="./../guides/slack.html">Slack</a></li><li><a href="./../reference/agent-config.html">Agent config</a></li><li><a href="./../reference/skills.html">Skills</a></li><li><a href="./../reference/sessions.html">Sessions and streaming</a></li><li><a href="./../evals.html">Evals</a></li></ul>`,49)])])}const u=a(i,[["render",n]]);export{k as __pageData,u as default};

package/dist/docs/assets/{example-agents_benny.md.C40vHRLc.lean.js → example-agents_benny.md.B0kwY7D_.lean.js}

@@ -1 +1 @@
-import{_ as a,c as t,o as s,ag as i}from"./chunks/framework.BCISBCiQ.js";const k=JSON.parse('{"title":"Route Slack work through repository playbooks","description":"Combine account-linked chat, allowlisted Socket Mode channel watching, inherited repository skills, and a custom local workspace.","frontmatter":{"title":"Route Slack work through repository playbooks","description":"Combine account-linked chat, allowlisted Socket Mode channel watching, inherited repository skills, and a custom local workspace."},"headers":[],"relativePath":"example-agents/benny.md","filePath":"example-agents/benny.md"}'),o={name:"example-agents/benny.md"};function n(l,e,r,h,d,p){return s(),t("div",null,[...e[0]||(e[0]=[i("",51)])])}const u=a(o,[["render",n]]);export{k as __pageData,u as default};
+import{_ as a,c as t,o as s,ag as o}from"./chunks/framework.BCISBCiQ.js";const k=JSON.parse('{"title":"Route Slack work through repository playbooks","description":"Combine account-linked chat, allowlisted Socket Mode channel watching, inherited repository skills, and a custom local workspace.","frontmatter":{"title":"Route Slack work through repository playbooks","description":"Combine account-linked chat, allowlisted Socket Mode channel watching, inherited repository skills, and a custom local workspace."},"headers":[],"relativePath":"example-agents/benny.md","filePath":"example-agents/benny.md"}'),i={name:"example-agents/benny.md"};function n(l,e,r,h,d,c){return s(),t("div",null,[...e[0]||(e[0]=[o("",49)])])}const u=a(i,[["render",n]]);export{k as __pageData,u as default};

package/dist/docs/assets/{example-agents_codebase-wiki.md.Dftj_tPp.js → example-agents_codebase-wiki.md.BBNw9Ekr.js}

@@ -1,8 +1,8 @@
-import{_ as s,c as a,o as t,ag as i}from"./chunks/framework.BCISBCiQ.js";const g=JSON.parse('{"title":"Build a feature wiki from merged pull requests","description":"Ingest every merged PR into per-feature wiki pages, consolidate with a daily digest schedule, and answer codebase questions with page citations.","frontmatter":{"title":"Build a feature wiki from merged pull requests","description":"Ingest every merged PR into per-feature wiki pages, consolidate with a daily digest schedule, and answer codebase questions with page citations."},"headers":[],"relativePath":"example-agents/codebase-wiki.md","filePath":"example-agents/codebase-wiki.md"}'),n={name:"example-agents/codebase-wiki.md"};function d(o,e,l,r,h,p){return t(),a("div",null,[...e[0]||(e[0]=[i(`<h1 id="build-a-feature-wiki-from-merged-pull-requests" tabindex="-1">Build a feature wiki from merged pull requests <a class="header-anchor" href="#build-a-feature-wiki-from-merged-pull-requests" aria-label="Permalink to &quot;Build a feature wiki from merged pull requests&quot;">​</a></h1><p>Codebase wiki keeps a living, feature-organized wiki of a repository. The GitHub channel acknowledges every closed pull request instantly, fetches a compact digest on the host, and spends a model turn only on merged PRs. The turn maps the change onto feature pages; a daily schedule writes a digest of what changed and rebuilds the index. Chat sessions answer codebase questions from the wiki with page citations.</p><p>Use this project when documentation should accumulate from merges instead of being regenerated from scratch. Use <a href="./knowledge-base.html">Knowledge base</a> when people should curate organizational context through conversation.</p><p><a href="./../../examples/codebase-wiki/">Browse the codebase wiki source.</a></p><h2 id="treat-prs-as-evidence-and-features-as-pages" tabindex="-1">Treat PRs as evidence and features as pages <a class="header-anchor" href="#treat-prs-as-evidence-and-features-as-pages" aria-label="Permalink to &quot;Treat PRs as evidence and features as pages&quot;">​</a></h2><p>The wiki refuses to become a merge log:</p><ul><li>The page tree is rigid: <code>index</code>, <code>features/&lt;slug&gt;</code>, and <code>digests/&lt;yyyy-mm-dd&gt;</code>. The store rejects anything else, so the wiki can&#39;t sprawl.</li><li>The <code>feature-mapping</code> skill requires a <code>wiki_search</code> before every write. A PR updates the page that owns its feature; a new page needs a genuinely new feature; chores change nothing.</li><li>Every touched page gets a dated changelog entry citing the PR number, so each fact traces back to a merge.</li></ul><p>The wiki itself is markdown on the serve host, in <code>.agent-serve/wiki/</code> by default with a <code>CODEBASE_WIKI_DIR</code> override. Sessions are disposable; the wiki is the durable state.</p><h2 id="follow-a-merged-pr" tabindex="-1">Follow a merged PR <a class="header-anchor" href="#follow-a-merged-pr" aria-label="Permalink to &quot;Follow a merged PR&quot;">​</a></h2><ol><li>GitHub delivers <code>pull_request</code> with action <code>closed</code>. The channel returns a task acknowledgement immediately.</li><li>The task fetches the digest with the host <code>gh</code> CLI: title, body, labels, changed files, and a bounded diff excerpt. No checkout.</li><li>The webhook payload can&#39;t say whether the PR merged, so the host checks <code>mergedAt</code> and skips abandoned PRs without a model turn.</li><li>For merged PRs, the task starts the turn with <code>pr/DIGEST.md</code> seeded through <code>workspaceFiles</code> and a <code>pr:&lt;owner/repo#N&gt;</code> continuation token, so redeliveries resume instead of double-ingesting.</li><li>The model follows <code>feature-mapping</code>: search, update or create feature pages, add changelog entries, and refresh <code>index</code> when pages were added.</li></ol><p>In chat, &quot;ingest PR #123&quot; runs the same flow through the <code>ingest_pr</code> tool, which writes the digest into the active session workspace.</p><h2 id="map-the-wiki-files" tabindex="-1">Map the wiki files <a class="header-anchor" href="#map-the-wiki-files" aria-label="Permalink to &quot;Map the wiki files&quot;">​</a></h2><table tabindex="0"><thead><tr><th>File</th><th>Purpose</th></tr></thead><tbody><tr><td><a href="../../examples/codebase-wiki/agent/agent.ts"><code>agent/agent.ts</code></a></td><td>Selects the local runtime and model.</td></tr><tr><td><a href="./../../examples/codebase-wiki/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Splits the job into merge ingestion and wiki-cited Q&amp;A.</td></tr><tr><td><a href="../../examples/codebase-wiki/agent/lib/wiki-store.ts"><code>agent/lib/wiki-store.ts</code></a></td><td>Enforces the rigid page tree and owns reads, writes, and search.</td></tr><tr><td><a href="../../examples/codebase-wiki/agent/lib/pr-digest.ts"><code>agent/lib/pr-digest.ts</code></a></td><td>Fetches PR metadata and diff, and formats <code>pr/DIGEST.md</code>.</td></tr><tr><td><a href="../../examples/codebase-wiki/agent/tools/ingest_pr.ts"><code>agent/tools/ingest_pr.ts</code></a></td><td>Exposes host digest preparation for chat-driven backfills.</td></tr><tr><td><a href="../../examples/codebase-wiki/agent/tools/wiki_read.ts"><code>agent/tools/wiki_read.ts</code></a>, <a href="../../examples/codebase-wiki/agent/tools/wiki_search.ts"><code>wiki_search.ts</code></a>, <a href="../../examples/codebase-wiki/agent/tools/wiki_write.ts"><code>wiki_write.ts</code></a></td><td>Read, search, and rewrite wiki pages.</td></tr><tr><td><a href="./../../examples/codebase-wiki/agent/skills/feature-mapping.html"><code>agent/skills/feature-mapping.md</code></a></td><td>Maps changes onto features and fixes the page and changelog shape.</td></tr><tr><td><a href="./../../examples/codebase-wiki/agent/schedules/daily-digest.html"><code>agent/schedules/daily-digest.md</code></a></td><td>Writes <code>digests/&lt;date&gt;</code>, rebuilds the index, and flags stale pages.</td></tr><tr><td><a href="../../examples/codebase-wiki/agent/channels/github.ts"><code>agent/channels/github.ts</code></a></td><td>Acknowledges closed PRs and starts merged-only ingest turns.</td></tr><tr><td><a href="../../examples/codebase-wiki/agent/storage.ts"><code>agent/storage.ts</code></a></td><td>Persists sessions and events with <code>cursorHostedStorage</code>.</td></tr><tr><td><a href="../../examples/codebase-wiki/evals/evals.config.ts"><code>evals/evals.config.ts</code></a></td><td>Caps eval run concurrency.</td></tr><tr><td><a href="../../examples/codebase-wiki/evals/ingest.eval.ts"><code>evals/ingest.eval.ts</code></a></td><td>Gates ingest decisions against the wiki filesystem.</td></tr></tbody></table><p>There is no MCP connection, subagent, hook, or A/B experiment.</p><h2 id="prepare-credentials-and-services" tabindex="-1">Prepare credentials and services <a class="header-anchor" href="#prepare-credentials-and-services" aria-label="Permalink to &quot;Prepare credentials and services&quot;">​</a></h2><p>You need:</p><ul><li>Node 22.13 or newer.</li><li>An agent-runtime credential for model turns.</li><li><code>gh</code> on <code>PATH</code> with read access to the PRs you ingest.</li></ul><p>The channel verifies webhook signatures when <code>GITHUB_WEBHOOK_SECRET</code> is set and narrows repositories with <code>CODEBASE_WIKI_REPOS=owner/repo,owner/other</code>. The agent never writes to GitHub. Its only side effects are wiki files on the serve host.</p><h2 id="validate-the-surface" tabindex="-1">Validate the surface <a class="header-anchor" href="#validate-the-surface" aria-label="Permalink to &quot;Validate the surface&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/codebase-wiki</span></span>
+import{_ as a,c as s,o as t,ag as i}from"./chunks/framework.BCISBCiQ.js";const g=JSON.parse('{"title":"Build a feature wiki from merged pull requests","description":"Ingest every merged PR into per-feature wiki pages, consolidate with a daily digest schedule, and answer codebase questions with page citations.","frontmatter":{"title":"Build a feature wiki from merged pull requests","description":"Ingest every merged PR into per-feature wiki pages, consolidate with a daily digest schedule, and answer codebase questions with page citations."},"headers":[],"relativePath":"example-agents/codebase-wiki.md","filePath":"example-agents/codebase-wiki.md"}'),n={name:"example-agents/codebase-wiki.md"};function d(o,e,r,l,h,c){return t(),s("div",null,[...e[0]||(e[0]=[i(`<h1 id="build-a-feature-wiki-from-merged-pull-requests" tabindex="-1">Build a feature wiki from merged pull requests <a class="header-anchor" href="#build-a-feature-wiki-from-merged-pull-requests" aria-label="Permalink to &quot;Build a feature wiki from merged pull requests&quot;">​</a></h1><p>Codebase wiki keeps a living, feature-organized wiki of a repository. The GitHub channel acknowledges every closed pull request instantly, fetches a compact digest on the host, and spends a model turn only on merged PRs. The turn maps the change onto feature pages; a daily schedule writes a digest of what changed and rebuilds the index. Chat sessions answer codebase questions from the wiki with page citations.</p><p>Use this project when documentation should accumulate from merges instead of being regenerated from scratch. Use <a href="./knowledge-base.html">Knowledge base</a> when people should curate organizational context through conversation.</p><p><a href="./../../examples/codebase-wiki/">Browse the codebase wiki source.</a></p><h2 id="treat-prs-as-evidence-and-features-as-pages" tabindex="-1">Treat PRs as evidence and features as pages <a class="header-anchor" href="#treat-prs-as-evidence-and-features-as-pages" aria-label="Permalink to &quot;Treat PRs as evidence and features as pages&quot;">​</a></h2><p>The wiki refuses to become a merge log:</p><ul><li>The page tree is rigid: <code>index</code>, <code>features/&lt;slug&gt;</code>, and <code>digests/&lt;yyyy-mm-dd&gt;</code>. The store rejects anything else, so the wiki can&#39;t sprawl.</li><li>The <code>feature-mapping</code> skill requires a <code>wiki_search</code> before every write. A PR updates the page that owns its feature; a new page needs a genuinely new feature; chores change nothing.</li><li>Every touched page gets a dated changelog entry citing the PR number, so each fact traces back to a merge.</li></ul><p>The wiki itself is markdown on the serve host, in a wiki directory by default with a <code>CODEBASE_WIKI_DIR</code> override. Sessions are disposable; the wiki is the durable state.</p><h2 id="follow-a-merged-pr" tabindex="-1">Follow a merged PR <a class="header-anchor" href="#follow-a-merged-pr" aria-label="Permalink to &quot;Follow a merged PR&quot;">​</a></h2><ol><li>GitHub delivers <code>pull_request</code> with action <code>closed</code>. The channel returns a task acknowledgement immediately.</li><li>The task fetches the digest with the host <code>gh</code> CLI: title, body, labels, changed files, and a bounded diff excerpt. No checkout.</li><li>The webhook payload can&#39;t say whether the PR merged, so the host checks <code>mergedAt</code> and skips abandoned PRs without a model turn.</li><li>For merged PRs, the task starts the turn with <code>pr/DIGEST.md</code> seeded through <code>workspaceFiles</code> and a <code>pr:&lt;owner/repo#N&gt;</code> continuation token, so redeliveries resume instead of double-ingesting.</li><li>The model follows <code>feature-mapping</code>: search, update or create feature pages, add changelog entries, and refresh <code>index</code> when pages were added.</li></ol><p>In chat, &quot;ingest PR #123&quot; runs the same flow through the <code>ingest_pr</code> tool, which writes the digest into the active session workspace.</p><h2 id="map-the-wiki-files" tabindex="-1">Map the wiki files <a class="header-anchor" href="#map-the-wiki-files" aria-label="Permalink to &quot;Map the wiki files&quot;">​</a></h2><table tabindex="0"><thead><tr><th>File</th><th>Purpose</th></tr></thead><tbody><tr><td><a href="../../examples/codebase-wiki/agent/agent.ts"><code>agent/agent.ts</code></a></td><td>Selects the local runtime and model.</td></tr><tr><td><a href="./../../examples/codebase-wiki/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Splits the job into merge ingestion and wiki-cited Q&amp;A.</td></tr><tr><td><a href="../../examples/codebase-wiki/agent/lib/wiki-store.ts"><code>agent/lib/wiki-store.ts</code></a></td><td>Enforces the rigid page tree and owns reads, writes, and search.</td></tr><tr><td><a href="../../examples/codebase-wiki/agent/lib/pr-digest.ts"><code>agent/lib/pr-digest.ts</code></a></td><td>Fetches PR metadata and diff, and formats <code>pr/DIGEST.md</code>.</td></tr><tr><td><a href="../../examples/codebase-wiki/agent/tools/ingest_pr.ts"><code>agent/tools/ingest_pr.ts</code></a></td><td>Exposes host digest preparation for chat-driven backfills.</td></tr><tr><td><a href="../../examples/codebase-wiki/agent/tools/wiki_read.ts"><code>agent/tools/wiki_read.ts</code></a>, <a href="../../examples/codebase-wiki/agent/tools/wiki_search.ts"><code>wiki_search.ts</code></a>, <a href="../../examples/codebase-wiki/agent/tools/wiki_write.ts"><code>wiki_write.ts</code></a></td><td>Read, search, and rewrite wiki pages.</td></tr><tr><td><a href="./../../examples/codebase-wiki/agent/skills/feature-mapping.html"><code>agent/skills/feature-mapping.md</code></a></td><td>Maps changes onto features and fixes the page and changelog shape.</td></tr><tr><td><a href="./../../examples/codebase-wiki/agent/schedules/daily-digest.html"><code>agent/schedules/daily-digest.md</code></a></td><td>Writes <code>digests/&lt;date&gt;</code>, rebuilds the index, and flags stale pages.</td></tr><tr><td><a href="../../examples/codebase-wiki/agent/channels/github.ts"><code>agent/channels/github.ts</code></a></td><td>Acknowledges closed PRs and starts merged-only ingest turns.</td></tr><tr><td><a href="../../examples/codebase-wiki/agent/storage.ts"><code>agent/storage.ts</code></a></td><td>Persists sessions and events with <code>cursorHostedStorage</code>.</td></tr><tr><td><a href="../../examples/codebase-wiki/evals/evals.config.ts"><code>evals/evals.config.ts</code></a></td><td>Caps eval run concurrency.</td></tr><tr><td><a href="../../examples/codebase-wiki/evals/ingest.eval.ts"><code>evals/ingest.eval.ts</code></a></td><td>Gates ingest decisions against the wiki filesystem.</td></tr></tbody></table><p>There is no MCP connection, subagent, hook, or A/B experiment.</p><h2 id="prepare-credentials-and-services" tabindex="-1">Prepare credentials and services <a class="header-anchor" href="#prepare-credentials-and-services" aria-label="Permalink to &quot;Prepare credentials and services&quot;">​</a></h2><p>You need:</p><ul><li>Node 22.13 or newer.</li><li>An agent-runtime credential for model turns.</li><li><code>gh</code> on <code>PATH</code> with read access to the PRs you ingest.</li></ul><p>The channel verifies webhook signatures when <code>GITHUB_WEBHOOK_SECRET</code> is set and narrows repositories with <code>CODEBASE_WIKI_REPOS=owner/repo,owner/other</code>. The agent never writes to GitHub. Its only side effects are wiki files on the serve host.</p><h2 id="validate-the-surface" tabindex="-1">Validate the surface <a class="header-anchor" href="#validate-the-surface" aria-label="Permalink to &quot;Validate the surface&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/codebase-wiki</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> info</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/codebase-wiki</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p>The manifest should report four server tools, one skill, one schedule, and the authored GitHub channel.</p><h2 id="ingest-without-webhook-plumbing" tabindex="-1">Ingest without webhook plumbing <a class="header-anchor" href="#ingest-without-webhook-plumbing" aria-label="Permalink to &quot;Ingest without webhook plumbing&quot;">​</a></h2><p>Replay a real merged PR as a closed delivery:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/codebase-wiki</span></span>
 <span class="line"></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> github</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> replay</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> https://github.com/owner/repo/pull/123</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
-<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/codebase-wiki</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --action</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> closed</span></span></code></pre></div><p>The reply is a 202 acknowledgement; the ingest continues in the task. Watch the session in the playground, then read the result on disk:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">ls</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/codebase-wiki/.agent-serve/wiki/features/</span></span></code></pre></div><p>Each ingested feature page carries an overview, a &quot;How it works&quot; section, and a changelog line citing the PR. Deterministic digest preparation works without a model turn:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> call</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> ingest_pr</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/codebase-wiki</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --action</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> closed</span></span></code></pre></div><p>The reply is a 202 acknowledgement; the ingest continues in the task. Watch the session in the playground, then open the wiki directory on the serve host. Feature pages land under <code>features/</code>.</p><p>Each ingested feature page carries an overview, a &quot;How it works&quot; section, and a changelog line citing the PR. Deterministic digest preparation works without a model turn:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> call</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> ingest_pr</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/codebase-wiki</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  --input</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &#39;{&quot;pr&quot;:&quot;https://github.com/owner/repo/pull/123&quot;}&#39;</span></span></code></pre></div><p>A PR closed without merging returns <code>merged: false</code> and a note telling the model to change nothing.</p><h2 id="run-the-daily-digest" tabindex="-1">Run the daily digest <a class="header-anchor" href="#run-the-daily-digest" aria-label="Permalink to &quot;Run the daily digest&quot;">​</a></h2><p>The schedule fires at 07:00 UTC. Under <code>agent-sdk dev</code>, trigger it by hand:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">curl</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -s</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -X</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> POST</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> http://127.0.0.1:3000/codebase-wiki/v1/dev/schedules/daily-digest</span></span></code></pre></div><p>The turn reads every feature changelog, writes <code>digests/&lt;today&gt;</code> grouped by feature with PR citations, rebuilds <code>index</code>, and reports one line per page it wrote. Entries dated today always count; a digest only claims a quiet day when no entry qualifies.</p><h2 id="run-the-evals" tabindex="-1">Run the evals <a class="header-anchor" href="#run-the-evals" aria-label="Permalink to &quot;Run the evals&quot;">​</a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/codebase-wiki</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/codebase-wiki</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> ingest/update-existing</span></span></code></pre></div><p>The cases seed a temp wiki through <code>CODEBASE_WIKI_DIR</code> and build digests with the same formatter the channel uses, so they run without GitHub or network access. The gates check the filesystem, not prose: a new feature page lands on a new slug, a related PR updates the existing page instead of duplicating it, an unmerged PR changes nothing, and the daily pass writes a digest naming both seeded features.</p><h2 id="reuse-the-merge-ingestion-pattern" tabindex="-1">Reuse the merge-ingestion pattern <a class="header-anchor" href="#reuse-the-merge-ingestion-pattern" aria-label="Permalink to &quot;Reuse the merge-ingestion pattern&quot;">​</a></h2><p>Copy this shape when events should accumulate into curated state:</p><ul><li>Acknowledge webhooks with a task and decide host-side whether a model turn is worth spending.</li><li>Seed evidence through <code>workspaceFiles</code> so the model never fetches.</li><li>Constrain the durable store&#39;s shape in code and its content in a skill.</li><li>Add a consolidation schedule so incremental writes stay coherent.</li></ul><h2 id="where-to-go-next" tabindex="-1">Where to go next <a class="header-anchor" href="#where-to-go-next" aria-label="Permalink to &quot;Where to go next&quot;">​</a></h2><ul><li><a href="./../guides/github.html">GitHub webhooks</a></li><li><a href="./../reference/schedules.html">Schedules</a></li><li><a href="./../reference/tools.html">Tools</a></li><li><a href="./../evals.html">Evals</a></li></ul>`,41)])])}const k=s(n,[["render",d]]);export{g as __pageData,k as default};
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/codebase-wiki</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> ingest/update-existing</span></span></code></pre></div><p>The cases seed a temp wiki through <code>CODEBASE_WIKI_DIR</code> and build digests with the same formatter the channel uses, so they run without GitHub or network access. The gates check the filesystem, not prose: a new feature page lands on a new slug, a related PR updates the existing page instead of duplicating it, an unmerged PR changes nothing, and the daily pass writes a digest naming both seeded features.</p><h2 id="reuse-the-merge-ingestion-pattern" tabindex="-1">Reuse the merge-ingestion pattern <a class="header-anchor" href="#reuse-the-merge-ingestion-pattern" aria-label="Permalink to &quot;Reuse the merge-ingestion pattern&quot;">​</a></h2><p>Copy this shape when events should accumulate into curated state:</p><ul><li>Acknowledge webhooks with a task and decide host-side whether a model turn is worth spending.</li><li>Seed evidence through <code>workspaceFiles</code> so the model never fetches.</li><li>Constrain the durable store&#39;s shape in code and its content in a skill.</li><li>Add a consolidation schedule so incremental writes stay coherent.</li></ul><h2 id="where-to-go-next" tabindex="-1">Where to go next <a class="header-anchor" href="#where-to-go-next" aria-label="Permalink to &quot;Where to go next&quot;">​</a></h2><ul><li><a href="./../guides/github.html">GitHub webhooks</a></li><li><a href="./../reference/schedules.html">Schedules</a></li><li><a href="./../reference/tools.html">Tools</a></li><li><a href="./../evals.html">Evals</a></li></ul>`,40)])])}const k=a(n,[["render",d]]);export{g as __pageData,k as default};

package/dist/docs/assets/{example-agents_codebase-wiki.md.Dftj_tPp.lean.js → example-agents_codebase-wiki.md.BBNw9Ekr.lean.js}

@@ -1 +1 @@
-import{_ as s,c as a,o as t,ag as i}from"./chunks/framework.BCISBCiQ.js";const g=JSON.parse('{"title":"Build a feature wiki from merged pull requests","description":"Ingest every merged PR into per-feature wiki pages, consolidate with a daily digest schedule, and answer codebase questions with page citations.","frontmatter":{"title":"Build a feature wiki from merged pull requests","description":"Ingest every merged PR into per-feature wiki pages, consolidate with a daily digest schedule, and answer codebase questions with page citations."},"headers":[],"relativePath":"example-agents/codebase-wiki.md","filePath":"example-agents/codebase-wiki.md"}'),n={name:"example-agents/codebase-wiki.md"};function d(o,e,l,r,h,p){return t(),a("div",null,[...e[0]||(e[0]=[i("",41)])])}const k=s(n,[["render",d]]);export{g as __pageData,k as default};
+import{_ as a,c as s,o as t,ag as i}from"./chunks/framework.BCISBCiQ.js";const g=JSON.parse('{"title":"Build a feature wiki from merged pull requests","description":"Ingest every merged PR into per-feature wiki pages, consolidate with a daily digest schedule, and answer codebase questions with page citations.","frontmatter":{"title":"Build a feature wiki from merged pull requests","description":"Ingest every merged PR into per-feature wiki pages, consolidate with a daily digest schedule, and answer codebase questions with page citations."},"headers":[],"relativePath":"example-agents/codebase-wiki.md","filePath":"example-agents/codebase-wiki.md"}'),n={name:"example-agents/codebase-wiki.md"};function d(o,e,r,l,h,c){return t(),s("div",null,[...e[0]||(e[0]=[i("",40)])])}const k=a(n,[["render",d]]);export{g as __pageData,k as default};

package/dist/docs/assets/{example-agents_concierge.md.MrKpQndp.js → example-agents_concierge.md.BzB2b20R.js}

@@ -2,9 +2,8 @@ import{_ as s,c as a,o as i,ag as t}from"./chunks/framework.BCISBCiQ.js";const k
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  agent: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;weather-agent&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  description:</span></span>
 <span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">    &quot;The weather-agent peer: delegate weather questions with ask; it runs its own tools (live Open-Meteo data) in its own context.&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p>The filename <a href="../../examples/concierge/agent/mcp-connections/weather.ts"><code>weather.ts</code></a> makes the MCP server name <code>weather</code>. The <code>agent</code> field points to the sibling project&#39;s mount slug.</p><p>This differs from a subagent. A peer keeps its own:</p><ul><li>root instructions,</li><li>tools and MCP connections,</li><li>durable sessions,</li><li>playground, and</li><li>public MCP endpoint.</li></ul><p>An SDK subagent inherits the parent&#39;s execution surface and only its parent can invoke it. See <a href="./../guides/agent-to-agent.html">Agent-to-agent</a> for the full comparison.</p><h2 id="follow-a-delegated-request" tabindex="-1">Follow a delegated request <a class="header-anchor" href="#follow-a-delegated-request" aria-label="Permalink to &quot;Follow a delegated request&quot;">​</a></h2><ol><li>A user asks Concierge what to pack for Paris.</li><li><a href="./../../examples/concierge/agent/instructions.html"><code>instructions.md</code></a> classifies packing advice as weather-related.</li><li>The model calls <code>weather.ask</code> with the city, timeframe, units, and the complete question.</li><li>The Agent SDK creates an MCP-channel session inside <code>weather-agent</code>.</li><li>Weather agent calls its own Open-Meteo tools and returns a reply.</li><li>If the turn exceeds the bounded MCP wait, <code>ask</code> returns <code>status: &quot;running&quot;</code>. Concierge calls <code>weather.check</code> with the returned <code>sessionId</code>.</li><li>Concierge relays the result and may add one sentence of travel advice.</li></ol><p>The weather session appears in the weather agent&#39;s playground. It doesn&#39;t share Concierge&#39;s conversation history.</p><h2 id="map-the-delegation-files" tabindex="-1">Map the delegation files <a class="header-anchor" href="#map-the-delegation-files" aria-label="Permalink to &quot;Map the delegation files&quot;">​</a></h2><table tabindex="0"><thead><tr><th>File</th><th>Purpose</th></tr></thead><tbody><tr><td><a href="../../examples/concierge/agent/agent.ts"><code>agent/agent.ts</code></a></td><td>Describes the root agent and selects the local runtime.</td></tr><tr><td><a href="./../../examples/concierge/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Draws a strict weather-only delegation boundary.</td></tr><tr><td><a href="../../examples/concierge/agent/mcp-connections/weather.ts"><code>agent/mcp-connections/weather.ts</code></a></td><td>Resolves the peer by its <code>weather-agent</code> slug.</td></tr><tr><td><a href="../../examples/concierge/agent/storage.ts"><code>agent/storage.ts</code></a></td><td>Persists sessions and events with <code>cursorHostedStorage</code>.</td></tr></tbody></table><p>Concierge doesn&#39;t author channels, tools, skills, subagents, schedules, hooks, A/B experiments, or evals. The built-in HTTP and MCP surfaces still exist.</p><p>Its own MCP endpoint exposes <code>ask</code> and <code>check</code>. It doesn&#39;t expose <code>call_tool</code> because Concierge has no server tools. The target weather agent does expose <code>call_tool</code>, so that tool also appears under Concierge&#39;s <code>weather</code> connection.</p><h2 id="mount-both-agents" tabindex="-1">Mount both agents <a class="header-anchor" href="#mount-both-agents" aria-label="Permalink to &quot;Mount both agents&quot;">​</a></h2><p>A peer can only resolve within a multi-agent serve host. Validating Concierge alone checks its files, but serving it alone fails because <code>weather-agent</code> isn&#39;t mounted.</p><p>From <code>packages/agent-serve</code>, validate both projects:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/concierge</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/weather-agent</span></span></code></pre></div><p>Don&#39;t serve the repository&#39;s whole <code>examples/</code> directory for this proof. Several advanced examples subscribe to live GitHub events. Create an ignored two-project mount instead. Copy only the authored files needed for this proof, leaving Weather&#39;s Slack channels out:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">mkdir</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -p</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$PWD</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/.agent-serve&quot;</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">PAIR_DIR</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$(</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">mktemp</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -d</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$PWD</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/.agent-serve/concierge-weather.XXXXXX&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p>The filename <a href="../../examples/concierge/agent/mcp-connections/weather.ts"><code>weather.ts</code></a> makes the MCP server name <code>weather</code>. The <code>agent</code> field points to the sibling project&#39;s mount slug.</p><p>This differs from a subagent. A peer keeps its own:</p><ul><li>root instructions,</li><li>tools and MCP connections,</li><li>durable sessions,</li><li>playground, and</li><li>public MCP endpoint.</li></ul><p>An SDK subagent inherits the parent&#39;s execution surface and only its parent can invoke it. See <a href="./../guides/agent-to-agent.html">Agent-to-agent</a> for the full comparison.</p><h2 id="follow-a-delegated-request" tabindex="-1">Follow a delegated request <a class="header-anchor" href="#follow-a-delegated-request" aria-label="Permalink to &quot;Follow a delegated request&quot;">​</a></h2><ol><li>A user asks Concierge what to pack for Paris.</li><li><a href="./../../examples/concierge/agent/instructions.html"><code>instructions.md</code></a> classifies packing advice as weather-related.</li><li>The model calls <code>weather.ask</code> with the city, timeframe, units, and the complete question.</li><li>The Agent SDK creates an MCP-channel session inside <code>weather-agent</code>.</li><li>Weather agent calls its own Open-Meteo tools and returns a reply.</li><li>If the turn exceeds the bounded MCP wait, <code>ask</code> returns <code>status: &quot;running&quot;</code>. Concierge calls <code>weather.check</code> with the returned <code>sessionId</code>.</li><li>Concierge relays the result and may add one sentence of travel advice.</li></ol><p>The weather session appears in the weather agent&#39;s playground. It doesn&#39;t share Concierge&#39;s conversation history.</p><h2 id="map-the-delegation-files" tabindex="-1">Map the delegation files <a class="header-anchor" href="#map-the-delegation-files" aria-label="Permalink to &quot;Map the delegation files&quot;">​</a></h2><table tabindex="0"><thead><tr><th>File</th><th>Purpose</th></tr></thead><tbody><tr><td><a href="../../examples/concierge/agent/agent.ts"><code>agent/agent.ts</code></a></td><td>Describes the root agent and selects the local runtime.</td></tr><tr><td><a href="./../../examples/concierge/agent/instructions.html"><code>agent/instructions.md</code></a></td><td>Draws a strict weather-only delegation boundary.</td></tr><tr><td><a href="../../examples/concierge/agent/mcp-connections/weather.ts"><code>agent/mcp-connections/weather.ts</code></a></td><td>Resolves the peer by its <code>weather-agent</code> slug.</td></tr><tr><td><a href="../../examples/concierge/agent/storage.ts"><code>agent/storage.ts</code></a></td><td>Persists sessions and events with <code>cursorHostedStorage</code>.</td></tr></tbody></table><p>Concierge doesn&#39;t author channels, tools, skills, subagents, schedules, hooks, A/B experiments, or evals. The built-in HTTP and MCP surfaces still exist.</p><p>Its own MCP endpoint exposes <code>ask</code> and <code>check</code>. It doesn&#39;t expose <code>call_tool</code> because Concierge has no server tools. The target weather agent does expose <code>call_tool</code>, so that tool also appears under Concierge&#39;s <code>weather</code> connection.</p><h2 id="mount-both-agents" tabindex="-1">Mount both agents <a class="header-anchor" href="#mount-both-agents" aria-label="Permalink to &quot;Mount both agents&quot;">​</a></h2><p>A peer can only resolve within a multi-agent serve host. Validating Concierge alone checks its files, but serving it alone fails because <code>weather-agent</code> isn&#39;t mounted.</p><p>From this package, validate both projects:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/concierge</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/weather-agent</span></span></code></pre></div><p>Don&#39;t serve the repository&#39;s whole <code>examples/</code> directory for this proof. Several advanced examples subscribe to live GitHub events. Create an ignored two-project mount instead. Copy only the authored files needed for this proof, leaving Weather&#39;s Slack channels out:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">PAIR_DIR</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$(</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">mktemp</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -d</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;\${</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">TMPDIR</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">:-/</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">tmp</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">}/concierge-weather.XXXXXX&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">mkdir</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -p</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$PAIR_DIR</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/concierge&quot;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$PAIR_DIR</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/weather-agent/agent&quot;</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">cp</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -R</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/concierge/agent</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$PAIR_DIR</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/concierge/&quot;</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">cp</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> examples/concierge/package.json</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$PAIR_DIR</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/concierge/&quot;</span></span>