@cursor/july 0.1.47 → 0.1.49

package/dist/bin/agent-serve.js

@@ -24,15 +24,17 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+import { spawn } from "node:child_process";
 import { readFile } from "node:fs/promises";
 import { resolve } from "node:path";
 import { parseArgs } from "node:util";
 import { formatDiagnostics, projectInfo, resolveAgentMounts, serve, } from "../index.js";
 import { registerAuthoredModuleLoaders } from "../internal/authored-loaders.js";
 import { cmdCall, cmdChat, cmdCost, cmdEval, cmdEvalCancel, cmdEvalStatus, cmdInit, cmdLogs, cmdPlayground, cmdResume, cmdRun, cmdSession, cmdSessions, cmdTrajectory, } from "../internal/cli-ax.js";
-import { cmdDeploy, cmdDeploymentStatus, cmdDeployments, cmdRotatePodCredential, cmdRotateToken, cmdSecretsList, cmdSecretsSet, cmdSecretsUnset, cmdStop, } from "../internal/cli-deploy.js";
+import { cmdDelete, cmdDeploy, cmdDeploymentStatus, cmdDeployments, cmdRotatePodCredential, cmdRotateToken, cmdSecretsList, cmdSecretsSet, cmdSecretsUnset, cmdStop, } from "../internal/cli-deploy.js";
 import { CLI_COMMAND_NAME as CLI, PACKAGE_NAME, packageVersion, } from "../internal/distribution.js";
 import { loadAgentLocalEnv } from "../internal/local-env.js";
+import { ENTER_RESTART_EXIT_CODE, isServeRestartWorker, runEnterRestartSupervisor, } from "../internal/serve-restart.js";
 import { clearTerminal, stdoutPalette } from "../internal/terminal-style.js";
 import { cmdUpdate, warnIfOutdated } from "../internal/update-check.js";
 const HELP = `${CLI} — serve a filesystem-defined Cursor agent over channels
@@ -104,6 +106,7 @@ Usage:
   ${CLI} deployments [--team <id>] [--json]
   ${CLI} deployment  <slug> [--team <id>] [--json]
   ${CLI} stop        <slug> [--team <id>] [--json] [--no-wait]
+  ${CLI} delete      <slug> [--team <id>] [--json] [--no-wait]
   ${CLI} rotate-token <slug> [--team <id>]
   ${CLI} rotate-pod-credential <slug> [--team <id>]
   ${CLI} secrets     set <slug> NAME... | list <slug> | unset <slug> NAME
@@ -113,7 +116,8 @@ Usage:
 
 Commands:
   serve       Serve agents under /<slug>/... (index at /). On a TTY, press
-              Enter to restart (reloads agent code + Vite HMR child).
+              Enter to restart (re-executes this process so agent code, evals,
+              and Vite reload).
   dev         Local development: same as \`${CLI} serve --dev\` (manual
               schedule/reminder dispatch, unsigned GitHub loopback, Vite HMR).
   chat        Interactive REPL against a running server (follow-ups keep the
@@ -222,6 +226,8 @@ Commands:
   deployments List the team's deployments (slug, status, generation, kind).
   deployment  Show one deployment in full (status, lastError, engine info).
   stop        Stop a deployment and wait until it reports stopped.
+  delete      Delete a deployment (teardown + purge the record) and wait
+              until GET 404s. The slug is then free to redeploy.
   rotate-token Mint a new alias token for a deployment (shown once); the old
               token stops working immediately.
   rotate-pod-credential
@@ -520,7 +526,10 @@ function main(argv) {
         // Best-effort, cached, and silent on failure. `update` does its own fresh
         // lookup, --json runs stay quiet for machine consumers, and `mcp` skips it
         // so MCP client spawns stay fast (stdout there is the protocol wire).
-        if (command !== "update" && command !== "mcp" && values.json !== true) {
+        if (command !== "update" &&
+            command !== "mcp" &&
+            values.json !== true &&
+            !isServeRestartWorker()) {
             yield warnIfOutdated();
         }
         switch (command) {
@@ -663,6 +672,9 @@ function main(argv) {
             case "stop": {
                 return cmdStop(positionals[0], deployOptionsFromValues(values));
             }
+            case "delete": {
+                return cmdDelete(positionals[0], deployOptionsFromValues(values));
+            }
             case "rotate-token": {
                 return cmdRotateToken(positionals[0], deployOptionsFromValues(values));
             }
@@ -1032,6 +1044,14 @@ function main(argv) {
                     },
                 };
                 const enterToRestart = process.stdin.isTTY === true;
+                if (enterToRestart && !isServeRestartWorker()) {
+                    return runEnterRestartSupervisor({
+                        spawn: (command, args, spawnOptions) => spawn(command, args, spawnOptions),
+                        onRestart: () => {
+                            clearTerminal();
+                        },
+                    });
+                }
                 const bannerOpts = {
                     noPlayground: values["no-playground"] === true,
                     dev,
@@ -1047,9 +1067,9 @@ function main(argv) {
                     }
                     return next;
                 });
-                let handle = yield start();
+                const handle = yield start();
                 let closing = false;
-                let restarting = false;
+                let restartRequested = false;
                 yield new Promise((resolve) => {
                     const shutdown = () => {
                         if (closing) {
@@ -1067,37 +1087,17 @@ function main(argv) {
                     process.stdin.resume();
                     process.stdin.on("data", (chunk) => {
                         const text = typeof chunk === "string" ? chunk : chunk.toString("utf8");
-                        if (!isEnterOnly(text) || closing || restarting) {
+                        if (!isEnterOnly(text) || closing) {
                             return;
                         }
-                        restarting = true;
-                        // Close + re-serve reloads authored agent modules and respawns Vite.
-                        // Hold the "restarting…" frame briefly so a fast close doesn't look
-                        // like Enter did nothing.
+                        closing = true;
+                        restartRequested = true;
                         clearTerminal();
                         process.stdout.write(style.dim("restarting…\n"));
-                        void Promise.all([handle.close(), sleep(450)])
-                            .then(() => {
-                            clearTerminal();
-                            return start();
-                        })
-                            .then((next) => {
-                            handle = next;
-                        })
-                            .catch((error) => {
-                            var _a;
-                            process.stderr.write(`[agent-sdk] restart failed: ${error instanceof Error
-                                ? ((_a = error.stack) !== null && _a !== void 0 ? _a : error.message)
-                                : String(error)}\n`);
-                            closing = true;
-                            resolve();
-                        })
-                            .finally(() => {
-                            restarting = false;
-                        });
+                        void handle.close().finally(resolve);
                     });
                 });
-                return 0;
+                return restartRequested ? ENTER_RESTART_EXIT_CODE : 0;
             }
             default: {
                 process.stderr.write(`Unknown command "${command}"\n\n${HELP}`);
@@ -1215,11 +1215,6 @@ function section(title, items) {
 function isEnterOnly(text) {
     return text.length > 0 && [...text].every((ch) => ch === "\n" || ch === "\r");
 }
-function sleep(ms) {
-    return new Promise((resolve) => {
-        setTimeout(resolve, ms);
-    });
-}
 /**
  * `dev [path]` takes a positional project root; `serve` keeps `--dir` only.
  * Reject mixing a positional with an explicit `--dir`, and extra args on `dev`.

package/dist/docs/404.html

@@ -9,14 +9,14 @@
     <link rel="preload stylesheet" href="/docs/assets/style.CK3hwt6q.css" as="style">
     <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
     
-    <script type="module" src="/docs/assets/app.IEXEim5m.js"></script>
+    <script type="module" src="/docs/assets/app.BpWAgjaR.js"></script>
     <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
     <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
     <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
   </head>
   <body>
     <div id="app"></div>
-    <script>window.__VP_HASH_MAP__=JSON.parse("{\"ab.md\":\"hdxsvX5M\",\"building-with-agents.md\":\"PeZaZA1P\",\"concepts.md\":\"CqOsxbMU\",\"deployment.md\":\"Cdp4rTE4\",\"evals.md\":\"qeoGgPw0\",\"example-agents_approval-buddy.md\":\"CIiZ9coo\",\"example-agents_benny.md\":\"l7JTmm8X\",\"example-agents_bugbot.md\":\"Dp5JqHSQ\",\"example-agents_codebase-wiki.md\":\"D-lteFf0\",\"example-agents_codeowners-review.md\":\"BU2ZXLf-\",\"example-agents_concierge.md\":\"DA2al_NK\",\"example-agents_fsd.md\":\"DPz9ezO4\",\"example-agents_index.md\":\"xmJ87d_3\",\"example-agents_knowledge-base.md\":\"IneynQSR\",\"example-agents_oncall.md\":\"ZE0n6ZFN\",\"example-agents_security-reviewer.md\":\"Dkf1gyo6\",\"example-agents_slack-agent.md\":\"06jQXTAI\",\"example-agents_weather-agent.md\":\"DC3lINjo\",\"guides_agent-to-agent.md\":\"Bmbxy-FA\",\"guides_cloud-runtime.md\":\"V5igN4Sq\",\"guides_github.md\":\"B8QKtIpw\",\"guides_human-in-the-loop.md\":\"BWvT7UqY\",\"guides_mcp-oauth.md\":\"C7G7IykG\",\"guides_opentelemetry.md\":\"Csn7ZI25\",\"guides_slack.md\":\"zriQpU_9\",\"guides_webhooks.md\":\"DB-r_er9\",\"hillclimbing.md\":\"yXqdlv2R\",\"index.md\":\"COeoSkcF\",\"quickstart.md\":\"DZxBu44y\",\"reference_agent-config.md\":\"DHkkvcEZ\",\"reference_artifacts.md\":\"BGG4bZo-\",\"reference_channels.md\":\"MN8iDy8V\",\"reference_cli.md\":\"BDmjPPAm\",\"reference_connections.md\":\"CSQiNzre\",\"reference_hooks.md\":\"DJE5DXcT\",\"reference_http-api.md\":\"DGrw_wOu\",\"reference_instructions.md\":\"CgoV-YEb\",\"reference_playground.md\":\"CPZhfYaO\",\"reference_project-layout.md\":\"D3MdHM2z\",\"reference_prompt.md\":\"BaiweQxE\",\"reference_schedules.md\":\"gmfYzf_I\",\"reference_sessions.md\":\"DBVFi2Sx\",\"reference_skills.md\":\"B_jHN7JL\",\"reference_subagents.md\":\"DSrGLIuB\",\"reference_tools.md\":\"kJfJAP2m\",\"scaffolding-agents.md\":\"mkc3B_ZW\",\"storage.md\":\"CJYd5YmN\",\"templates_triage.md\":\"CConZt1z\",\"troubleshooting.md\":\"CkgwmVI0\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"Agent SDK\",\"description\":\"Filesystem-first framework for building and serving Cursor agents.\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"nav\":[{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Guides\",\"link\":\"/guides/webhooks\"},{\"text\":\"Examples\",\"link\":\"/example-agents/\"},{\"text\":\"Reference\",\"link\":\"/reference/project-layout\"}],\"search\":{\"provider\":\"local\"},\"outline\":{\"level\":[2,3]},\"sidebar\":[{\"text\":\"Getting started\",\"items\":[{\"text\":\"Overview\",\"link\":\"/\"},{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Scaffold an agent with Cursor\",\"link\":\"/scaffolding-agents\"},{\"text\":\"Concepts\",\"link\":\"/concepts\"}]},{\"text\":\"Templates\",\"items\":[{\"text\":\"Triage agent\",\"link\":\"/templates/triage\"}]},{\"text\":\"Improving agents\",\"items\":[{\"text\":\"Building agents with agents\",\"link\":\"/building-with-agents\"},{\"text\":\"Evals\",\"link\":\"/evals\"},{\"text\":\"Live A/B metrics\",\"link\":\"/ab\"},{\"text\":\"Storage\",\"link\":\"/storage\"},{\"text\":\"Hillclimbing\",\"link\":\"/hillclimbing\"}]},{\"text\":\"Guides\",\"items\":[{\"text\":\"Webhooks & custom channels\",\"link\":\"/guides/webhooks\"},{\"text\":\"GitHub\",\"link\":\"/guides/github\"},{\"text\":\"Slack\",\"link\":\"/guides/slack\"},{\"text\":\"Human-in-the-loop approvals\",\"link\":\"/guides/human-in-the-loop\"},{\"text\":\"Agent-to-agent\",\"link\":\"/guides/agent-to-agent\"},{\"text\":\"Host MCP OAuth\",\"link\":\"/guides/mcp-oauth\"},{\"text\":\"Cloud runtime\",\"link\":\"/guides/cloud-runtime\"},{\"text\":\"OpenTelemetry\",\"link\":\"/guides/opentelemetry\"}]},{\"text\":\"Example agents\",\"items\":[{\"text\":\"Choose an example\",\"link\":\"/example-agents/\"},{\"text\":\"Weather agent\",\"link\":\"/example-agents/weather-agent\"},{\"text\":\"Slack agent\",\"link\":\"/example-agents/slack-agent\"},{\"text\":\"Concierge\",\"link\":\"/example-agents/concierge\"},{\"text\":\"Playbook router\",\"link\":\"/example-agents/benny\"},{\"text\":\"Alert investigator\",\"link\":\"/example-agents/oncall\"},{\"text\":\"PR evidence reviewer\",\"link\":\"/example-agents/bugbot\"},{\"text\":\"Approval Buddy\",\"link\":\"/example-agents/approval-buddy\"},{\"text\":\"Security Reviewer\",\"link\":\"/example-agents/security-reviewer\"},{\"text\":\"Remote PR coordinator\",\"link\":\"/example-agents/fsd\"},{\"text\":\"Knowledge base\",\"link\":\"/example-agents/knowledge-base\"},{\"text\":\"Codebase wiki\",\"link\":\"/example-agents/codebase-wiki\"},{\"text\":\"Codeowners review\",\"link\":\"/example-agents/codeowners-review\"}]},{\"text\":\"Operating\",\"items\":[{\"text\":\"Deployment\",\"link\":\"/deployment\"},{\"text\":\"Fix common problems\",\"link\":\"/troubleshooting\"}]},{\"text\":\"Reference\",\"collapsed\":false,\"items\":[{\"text\":\"Project layout\",\"link\":\"/reference/project-layout\"},{\"text\":\"Agent config\",\"link\":\"/reference/agent-config\"},{\"text\":\"Instructions\",\"link\":\"/reference/instructions\"},{\"text\":\"Tools\",\"link\":\"/reference/tools\"},{\"text\":\"prompt\",\"link\":\"/reference/prompt\"},{\"text\":\"Skills\",\"link\":\"/reference/skills\"},{\"text\":\"MCP Connections\",\"link\":\"/reference/connections\"},{\"text\":\"Subagents\",\"link\":\"/reference/subagents\"},{\"text\":\"Channels\",\"link\":\"/reference/channels\"},{\"text\":\"Schedules & reminders\",\"link\":\"/reference/schedules\"},{\"text\":\"Hooks\",\"link\":\"/reference/hooks\"},{\"text\":\"Artifacts\",\"link\":\"/reference/artifacts\"},{\"text\":\"Sessions & streaming\",\"link\":\"/reference/sessions\"},{\"text\":\"Playground\",\"link\":\"/reference/playground\"},{\"text\":\"CLI\",\"link\":\"/reference/cli\"},{\"text\":\"HTTP API\",\"link\":\"/reference/http-api\"}]}]},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
+    <script>window.__VP_HASH_MAP__=JSON.parse("{\"ab.md\":\"DYjwREAP\",\"building-with-agents.md\":\"PeZaZA1P\",\"concepts.md\":\"CqOsxbMU\",\"deployment.md\":\"Cdp4rTE4\",\"evals.md\":\"CVe_O75-\",\"example-agents_approval-buddy.md\":\"CIiZ9coo\",\"example-agents_benny.md\":\"l7JTmm8X\",\"example-agents_bugbot.md\":\"Dp5JqHSQ\",\"example-agents_codebase-wiki.md\":\"D-lteFf0\",\"example-agents_codeowners-review.md\":\"BU2ZXLf-\",\"example-agents_concierge.md\":\"DA2al_NK\",\"example-agents_fsd.md\":\"DPz9ezO4\",\"example-agents_index.md\":\"xmJ87d_3\",\"example-agents_knowledge-base.md\":\"IneynQSR\",\"example-agents_oncall.md\":\"ZE0n6ZFN\",\"example-agents_security-reviewer.md\":\"Dkf1gyo6\",\"example-agents_slack-agent.md\":\"06jQXTAI\",\"example-agents_weather-agent.md\":\"DC3lINjo\",\"guides_agent-to-agent.md\":\"Bmbxy-FA\",\"guides_cloud-runtime.md\":\"V5igN4Sq\",\"guides_github.md\":\"B8QKtIpw\",\"guides_human-in-the-loop.md\":\"BWvT7UqY\",\"guides_mcp-oauth.md\":\"C7G7IykG\",\"guides_opentelemetry.md\":\"Csn7ZI25\",\"guides_slack.md\":\"zriQpU_9\",\"guides_webhooks.md\":\"DB-r_er9\",\"hillclimbing.md\":\"yXqdlv2R\",\"index.md\":\"COeoSkcF\",\"quickstart.md\":\"DZxBu44y\",\"reference_agent-config.md\":\"DHkkvcEZ\",\"reference_artifacts.md\":\"BGG4bZo-\",\"reference_channels.md\":\"MN8iDy8V\",\"reference_cli.md\":\"BDmjPPAm\",\"reference_connections.md\":\"CSQiNzre\",\"reference_hooks.md\":\"DJE5DXcT\",\"reference_http-api.md\":\"DMbdFGVQ\",\"reference_instructions.md\":\"CgoV-YEb\",\"reference_playground.md\":\"CPZhfYaO\",\"reference_project-layout.md\":\"D3MdHM2z\",\"reference_prompt.md\":\"BaiweQxE\",\"reference_schedules.md\":\"gmfYzf_I\",\"reference_sessions.md\":\"DBVFi2Sx\",\"reference_skills.md\":\"B_jHN7JL\",\"reference_subagents.md\":\"DSrGLIuB\",\"reference_tools.md\":\"kJfJAP2m\",\"scaffolding-agents.md\":\"mkc3B_ZW\",\"storage.md\":\"D8NQ389l\",\"templates_triage.md\":\"CConZt1z\",\"troubleshooting.md\":\"CkgwmVI0\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"Agent SDK\",\"description\":\"Filesystem-first framework for building and serving Cursor agents.\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"nav\":[{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Guides\",\"link\":\"/guides/webhooks\"},{\"text\":\"Examples\",\"link\":\"/example-agents/\"},{\"text\":\"Reference\",\"link\":\"/reference/project-layout\"}],\"search\":{\"provider\":\"local\"},\"outline\":{\"level\":[2,3]},\"sidebar\":[{\"text\":\"Getting started\",\"items\":[{\"text\":\"Overview\",\"link\":\"/\"},{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Scaffold an agent with Cursor\",\"link\":\"/scaffolding-agents\"},{\"text\":\"Concepts\",\"link\":\"/concepts\"}]},{\"text\":\"Templates\",\"items\":[{\"text\":\"Triage agent\",\"link\":\"/templates/triage\"}]},{\"text\":\"Improving agents\",\"items\":[{\"text\":\"Building agents with agents\",\"link\":\"/building-with-agents\"},{\"text\":\"Evals\",\"link\":\"/evals\"},{\"text\":\"Live A/B metrics\",\"link\":\"/ab\"},{\"text\":\"Storage\",\"link\":\"/storage\"},{\"text\":\"Hillclimbing\",\"link\":\"/hillclimbing\"}]},{\"text\":\"Guides\",\"items\":[{\"text\":\"Webhooks & custom channels\",\"link\":\"/guides/webhooks\"},{\"text\":\"GitHub\",\"link\":\"/guides/github\"},{\"text\":\"Slack\",\"link\":\"/guides/slack\"},{\"text\":\"Human-in-the-loop approvals\",\"link\":\"/guides/human-in-the-loop\"},{\"text\":\"Agent-to-agent\",\"link\":\"/guides/agent-to-agent\"},{\"text\":\"Host MCP OAuth\",\"link\":\"/guides/mcp-oauth\"},{\"text\":\"Cloud runtime\",\"link\":\"/guides/cloud-runtime\"},{\"text\":\"OpenTelemetry\",\"link\":\"/guides/opentelemetry\"}]},{\"text\":\"Example agents\",\"items\":[{\"text\":\"Choose an example\",\"link\":\"/example-agents/\"},{\"text\":\"Weather agent\",\"link\":\"/example-agents/weather-agent\"},{\"text\":\"Slack agent\",\"link\":\"/example-agents/slack-agent\"},{\"text\":\"Concierge\",\"link\":\"/example-agents/concierge\"},{\"text\":\"Playbook router\",\"link\":\"/example-agents/benny\"},{\"text\":\"Alert investigator\",\"link\":\"/example-agents/oncall\"},{\"text\":\"PR evidence reviewer\",\"link\":\"/example-agents/bugbot\"},{\"text\":\"Approval Buddy\",\"link\":\"/example-agents/approval-buddy\"},{\"text\":\"Security Reviewer\",\"link\":\"/example-agents/security-reviewer\"},{\"text\":\"Remote PR coordinator\",\"link\":\"/example-agents/fsd\"},{\"text\":\"Knowledge base\",\"link\":\"/example-agents/knowledge-base\"},{\"text\":\"Codebase wiki\",\"link\":\"/example-agents/codebase-wiki\"},{\"text\":\"Codeowners review\",\"link\":\"/example-agents/codeowners-review\"}]},{\"text\":\"Operating\",\"items\":[{\"text\":\"Deployment\",\"link\":\"/deployment\"},{\"text\":\"Fix common problems\",\"link\":\"/troubleshooting\"}]},{\"text\":\"Reference\",\"collapsed\":false,\"items\":[{\"text\":\"Project layout\",\"link\":\"/reference/project-layout\"},{\"text\":\"Agent config\",\"link\":\"/reference/agent-config\"},{\"text\":\"Instructions\",\"link\":\"/reference/instructions\"},{\"text\":\"Tools\",\"link\":\"/reference/tools\"},{\"text\":\"prompt\",\"link\":\"/reference/prompt\"},{\"text\":\"Skills\",\"link\":\"/reference/skills\"},{\"text\":\"MCP Connections\",\"link\":\"/reference/connections\"},{\"text\":\"Subagents\",\"link\":\"/reference/subagents\"},{\"text\":\"Channels\",\"link\":\"/reference/channels\"},{\"text\":\"Schedules & reminders\",\"link\":\"/reference/schedules\"},{\"text\":\"Hooks\",\"link\":\"/reference/hooks\"},{\"text\":\"Artifacts\",\"link\":\"/reference/artifacts\"},{\"text\":\"Sessions & streaming\",\"link\":\"/reference/sessions\"},{\"text\":\"Playground\",\"link\":\"/reference/playground\"},{\"text\":\"CLI\",\"link\":\"/reference/cli\"},{\"text\":\"HTTP API\",\"link\":\"/reference/http-api\"}]}]},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
     
   </body>
 </html>

package/dist/docs/ab.html

@@ -9,11 +9,11 @@
     <link rel="preload stylesheet" href="/docs/assets/style.CK3hwt6q.css" as="style">
     <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
     
-    <script type="module" src="/docs/assets/app.IEXEim5m.js"></script>
+    <script type="module" src="/docs/assets/app.BpWAgjaR.js"></script>
     <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
-    <link rel="modulepreload" href="/docs/assets/chunks/theme.DroGdz_Q.js">
+    <link rel="modulepreload" href="/docs/assets/chunks/theme.B5IvUxOB.js">
     <link rel="modulepreload" href="/docs/assets/chunks/framework.CAZyNGu9.js">
-    <link rel="modulepreload" href="/docs/assets/ab.md.hdxsvX5M.lean.js">
+    <link rel="modulepreload" href="/docs/assets/ab.md.DYjwREAP.lean.js">
     <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
     <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
   </head>
@@ -66,13 +66,13 @@
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">  return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> conciseWeatherResult;</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span>
 <span class="line"></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> baselineWeatherResult;</span></span></code></pre></div><p>This makes the assignment available to deterministic code as well as the model prompt. Use both patterns together when one experiment must steer the prompt and host code at once.</p><p><code>defineAB</code> does not select a different model or runtime for each arm. Keep those settings in <code>agent/agent.ts</code>, or write explicit host logic when your experiment needs another behavior lever.</p><p>The split and selected arm can affect agent behavior. <code>derive</code> and <code>onSample</code> only observe the resulting event stream. Errors in either callback are logged and never fail the turn.</p><h2 id="collect-built-in-and-custom-metrics" tabindex="-1">Collect built-in and custom metrics <a class="header-anchor" href="#collect-built-in-and-custom-metrics" aria-label="Permalink to &quot;Collect built-in and custom metrics&quot;">​</a></h2><p>Metrics accumulate for each session and experiment. When one session joins several experiments, every enrolled experiment folds the same turn and tool events into its own counters.</p><table tabindex="0"><thead><tr><th>Metric</th><th>How the Agent SDK calculates it</th></tr></thead><tbody><tr><td><code>turns</code></td><td>Adds one on <code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>turnFailures</code></td><td>Adds one on <code>turn.failed</code></td></tr><tr><td><code>toolCalls</code></td><td>Adds one for each <code>action.result</code></td></tr><tr><td><code>toolErrors</code></td><td>Adds one when <code>action.result.data.isError</code> is true</td></tr><tr><td><code>inputTokens</code>, <code>outputTokens</code></td><td>Adds usage from completed turns</td></tr><tr><td><code>cacheReadTokens</code>, <code>cacheWriteTokens</code></td><td>Adds cache usage from completed turns</td></tr><tr><td><code>costUsd</code></td><td>Sums the estimated turn cost recorded on <code>turn.completed</code> (turns whose model has no known rates contribute 0)</td></tr><tr><td><code>wallTimeMs</code></td><td>Sums the time from <code>turn.started</code> to its completed or failed event</td></tr><tr><td><code>custom</code></td><td>Sums finite numeric deltas returned by <code>derive</code></td></tr></tbody></table><p><code>onSample</code> fires after every <code>turn.completed</code> and <code>turn.failed</code> event for an enrolled arm. The sample contains:</p><table tabindex="0"><thead><tr><th>Field</th><th>Value</th></tr></thead><tbody><tr><td><code>experiment</code></td><td>Experiment name</td></tr><tr><td><code>variant</code>, <code>variantLabel?</code></td><td>Sticky arm and optional display label</td></tr><tr><td><code>sessionId</code>, <code>channelId</code></td><td>Source session</td></tr><tr><td><code>metrics</code></td><td>Cumulative metrics through this turn</td></tr><tr><td><code>reason</code></td><td><code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>at</code></td><td>Terminal event timestamp</td></tr></tbody></table><p>The metrics are cumulative, not per-turn deltas. A second sample from the same session includes the first turn&#39;s counts.</p><p>Each <code>derive</code> extractor runs on every session event for its enrolled experiment, including streamed <code>message.appended</code> events. Keep it synchronous and cheap. Return a finite number to add a delta, or <code>null</code> to skip the event. Send samples to your metrics service from <code>onSample</code>; do not perform network or disk work in <code>derive</code>.</p><p>Skipped sessions never call <code>onSample</code>. Errors from <code>derive</code> or <code>onSample</code> are logged, then metric collection continues.</p><h2 id="inspect-assignments-and-results" tabindex="-1">Inspect assignments and results <a class="header-anchor" href="#inspect-assignments-and-results" aria-label="Permalink to &quot;Inspect assignments and results&quot;">​</a></h2><p>Open the playground&#39;s <strong>A/Bs</strong> tab to see aggregate arm totals and per-session assignments. The tab reads <code>GET /v1/abs</code>.</p><p>The response has two views of the same durable data:</p><table tabindex="0"><thead><tr><th>Field</th><th>Contents</th></tr></thead><tbody><tr><td><code>experiments</code></td><td>Declared variants, skipped-session count, arm session counts, and aggregate metrics</td></tr><tr><td><code>sessions</code></td><td>Visible sessions with their assignments and cumulative metrics</td></tr></tbody></table><p><code>GET /v1/abs</code> returns sessions visible to the current principal by default. In <code>--dev</code>, loopback requests include every session. Add <code>--allow-anonymous</code> to include every session from non-loopback callers too. This include-all behavior can still apply to <code>GET /v1/abs</code> in dev when bearer or custom auth keeps <code>GET /v1/sessions</code> owner-scoped.</p><p>Session <code>events.ndjson</code> is the source of truth for assignment + fold. <code>GET /v1/abs</code> recomputes aggregates from those logs. For durable sample and snapshot export, declare an <code>abs</code> table in <code>agent/storage.ts</code> — same idea as the eval-runs table. See <a href="./storage.html#eval-and-a-b-tables">Storage</a>.</p><h2 id="configure-the-playground-fold-window" tabindex="-1">Configure the playground fold window <a class="header-anchor" href="#configure-the-playground-fold-window" aria-label="Permalink to &quot;Configure the playground fold window&quot;">​</a></h2><p>Assignments and foldable metrics already persist in each session&#39;s <code>events.ndjson</code> under <code>--state-root</code>. The optional <code>agent/ab.config.ts</code> only caps how many sessions the playground and <code>GET /v1/abs</code> fold:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineABConfig } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/ab&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> baselineWeatherResult;</span></span></code></pre></div><p>This makes the assignment available to deterministic code as well as the model prompt. Use both patterns together when one experiment must steer the prompt and host code at once.</p><p><code>defineAB</code> does not select a different model or runtime for each arm. Keep those settings in <code>agent/agent.ts</code>, or write explicit host logic when your experiment needs another behavior lever.</p><p>The split and selected arm can affect agent behavior. <code>derive</code> and <code>onSample</code> only observe the resulting event stream. Errors in either callback are logged and never fail the turn.</p><h2 id="collect-built-in-and-custom-metrics" tabindex="-1">Collect built-in and custom metrics <a class="header-anchor" href="#collect-built-in-and-custom-metrics" aria-label="Permalink to &quot;Collect built-in and custom metrics&quot;">​</a></h2><p>Metrics accumulate for each session and experiment. When one session joins several experiments, every enrolled experiment folds the same turn and tool events into its own counters.</p><table tabindex="0"><thead><tr><th>Metric</th><th>How the Agent SDK calculates it</th></tr></thead><tbody><tr><td><code>turns</code></td><td>Adds one on <code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>turnFailures</code></td><td>Adds one on <code>turn.failed</code></td></tr><tr><td><code>toolCalls</code></td><td>Adds one for each <code>action.result</code></td></tr><tr><td><code>toolErrors</code></td><td>Adds one when <code>action.result.data.isError</code> is true</td></tr><tr><td><code>inputTokens</code>, <code>outputTokens</code></td><td>Adds usage from completed turns</td></tr><tr><td><code>cacheReadTokens</code>, <code>cacheWriteTokens</code></td><td>Adds cache usage from completed turns</td></tr><tr><td><code>costUsd</code></td><td>Sums the estimated turn cost recorded on <code>turn.completed</code> (turns whose model has no known rates contribute 0)</td></tr><tr><td><code>wallTimeMs</code></td><td>Sums the time from <code>turn.started</code> to its completed or failed event</td></tr><tr><td><code>custom</code></td><td>Sums finite numeric deltas returned by <code>derive</code></td></tr></tbody></table><p><code>onSample</code> fires after every <code>turn.completed</code> and <code>turn.failed</code> event for an enrolled arm. The sample contains:</p><table tabindex="0"><thead><tr><th>Field</th><th>Value</th></tr></thead><tbody><tr><td><code>experiment</code></td><td>Experiment name</td></tr><tr><td><code>variant</code>, <code>variantLabel?</code></td><td>Sticky arm and optional display label</td></tr><tr><td><code>sessionId</code>, <code>channelId</code></td><td>Source session</td></tr><tr><td><code>metrics</code></td><td>Cumulative metrics through this turn</td></tr><tr><td><code>reason</code></td><td><code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>at</code></td><td>Terminal event timestamp</td></tr></tbody></table><p>The metrics are cumulative, not per-turn deltas. A second sample from the same session includes the first turn&#39;s counts.</p><p>Each <code>derive</code> extractor runs on every session event for its enrolled experiment, including streamed <code>message.appended</code> events. Keep it synchronous and cheap. Return a finite number to add a delta, or <code>null</code> to skip the event. Send samples to your metrics service from <code>onSample</code>; do not perform network or disk work in <code>derive</code>.</p><p>Skipped sessions never call <code>onSample</code>. Errors from <code>derive</code> or <code>onSample</code> are logged, then metric collection continues.</p><h2 id="inspect-assignments-and-results" tabindex="-1">Inspect assignments and results <a class="header-anchor" href="#inspect-assignments-and-results" aria-label="Permalink to &quot;Inspect assignments and results&quot;">​</a></h2><p>Open the playground&#39;s <strong>A/Bs</strong> tab to see aggregate arm totals and per-session assignments. The tab reads <code>GET /v1/abs</code>.</p><p>The response has two views of the same durable data:</p><table tabindex="0"><thead><tr><th>Field</th><th>Contents</th></tr></thead><tbody><tr><td><code>experiments</code></td><td>Declared variants, skipped-session count, arm session counts, and aggregate metrics</td></tr><tr><td><code>sessions</code></td><td>Visible sessions with their assignments and cumulative metrics</td></tr></tbody></table><p><code>GET /v1/abs</code> returns sessions visible to the current principal by default. In <code>--dev</code>, loopback requests include every session. Add <code>--allow-anonymous</code> to include every session from non-loopback callers too. This include-all behavior can still apply to <code>GET /v1/abs</code> in dev when bearer or custom auth keeps <code>GET /v1/sessions</code> owner-scoped.</p><p>Session <code>events.ndjson</code> is the source of truth for assignment + fold. <code>GET /v1/abs</code> recomputes aggregates from those logs. Any <code>agent/storage.ts</code> exports samples and snapshots durably: an authored <code>abs</code> table when the backend has a native shape for it, or the table derived over the KV core otherwise. See <a href="./storage.html#eval-and-a-b-tables">Storage</a>.</p><h2 id="configure-the-playground-fold-window" tabindex="-1">Configure the playground fold window <a class="header-anchor" href="#configure-the-playground-fold-window" aria-label="Permalink to &quot;Configure the playground fold window&quot;">​</a></h2><p>Assignments and foldable metrics already persist in each session&#39;s <code>events.ndjson</code> under <code>--state-root</code>. The optional <code>agent/ab.config.ts</code> only caps how many sessions the playground and <code>GET /v1/abs</code> fold:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineABConfig } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/ab&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
 <span class="line"></span>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineABConfig</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
 <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">  // Optional — defaults to 200. Only affects GET /v1/abs / A/Bs tab.</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  maxPlaygroundSessions: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">500</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p><code>maxPlaygroundSessions</code> keeps the newest sessions in the fold. It does not prune session logs or change assignment. For export to S3, a DB, or your metrics vendor, send samples from <code>onSample</code> or declare a storage <code>abs</code> table.</p><h2 id="keep-assignments-durable" tabindex="-1">Keep assignments durable <a class="header-anchor" href="#keep-assignments-durable" aria-label="Permalink to &quot;Keep assignments durable&quot;">​</a></h2><p>The append-only <code>events.ndjson</code> stream is the source of truth. Each <code>ab.assigned</code> event persists a variant key or null skip. Built-in metrics come from the turn and tool events that follow it.</p><p>After a server restart or a parked session resumes, the live collector replays the stream to rebuild cumulative counters. Replay does not call <code>onSample</code> (or write to the storage <code>abs</code> table) for historical turns. Only a new completed or failed turn emits another sample.</p><p>The snapshot API also replays <code>derive</code> across the full stream, so custom totals match the current extractor. Changing a derive function can change historical snapshot totals. Treat metric definitions as versioned experiment code.</p><h2 id="keep-eval-traffic-separate" tabindex="-1">Keep eval traffic separate <a class="header-anchor" href="#keep-eval-traffic-separate" aria-label="Permalink to &quot;Keep eval traffic separate&quot;">​</a></h2><p>Sessions created by <code>agent-sdk eval</code> and the playground Evals runner use <code>purpose: &quot;eval&quot;</code>. They skip A/B enrollment entirely:</p><ul><li>No split function runs.</li><li>No <code>ab.assigned</code> event is recorded.</li><li>No <code>onSample</code> callback fires.</li><li>The session is omitted from <code>GET /v1/abs</code>.</li></ul><p>Ordinary chat, <code>agent-sdk run</code>, Slack, GitHub, and other channel sessions use the live purpose. You do not need <code>splitIf</code> to exclude eval traffic.</p><h2 id="know-the-boundaries" tabindex="-1">Know the boundaries <a class="header-anchor" href="#know-the-boundaries" aria-label="Permalink to &quot;Know the boundaries&quot;">​</a></h2><p><code>defineAB</code> provides sticky assignment, variant instructions, <code>session.abs</code> for tools, cumulative metrics, and local inspection. It does not provide:</p><ul><li>A test command, assertion API, or pass/fail result</li><li>Statistical significance calculations</li><li>An experiment rollout or lifecycle service</li><li>Per-variant model or runtime configuration</li><li>A built-in analytics warehouse (bring your own via <code>onSample</code> or the storage <code>abs</code> table)</li></ul><p>Use <a href="./evals.html">evals</a> to protect known behavior. Use <code>onSample</code> or a storage <code>abs</code> table when you need sample/snapshot exports beyond the session event log.</p><h2 id="what-s-next" tabindex="-1">What&#39;s next <a class="header-anchor" href="#what-s-next" aria-label="Permalink to &quot;What&#39;s next&quot;">​</a></h2><p>Continue with these pages:</p><ul><li><a href="./evals.html">Evals</a>: pass/fail regression checks on fixed inputs</li><li><a href="./hillclimbing.html">Hillclimbing</a>: improve an agent against fixed fixtures</li><li><a href="./reference/hooks.html">Hooks</a>: other event-stream consumers</li><li><a href="./reference/sessions.html">Sessions and streaming</a>: the <code>ab.assigned</code> event and durable log</li><li><a href="./reference/playground.html">Playground</a>: the A/Bs tab</li><li><a href="./reference/http-api.html">HTTP API</a>: <code>GET /v1/abs</code></li><li><a href="./../skills/ab/SKILL.html">Live A/B metrics skill</a>: have a coding agent wire an experiment</li></ul></div></div></main><footer class="VPDocFooter" data-v-d5d7ccd1 data-v-a5908b0f><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-a5908b0f><span class="visually-hidden" id="doc-footer-aria-label" data-v-a5908b0f>Pager</span><div class="pager" data-v-a5908b0f><a class="VPLink link pager-link prev" href="/docs/evals.html" data-v-a5908b0f><!--[--><span class="desc" data-v-a5908b0f>Previous page</span><span class="title" data-v-a5908b0f>Evals</span><!--]--></a></div><div class="pager" data-v-a5908b0f><a class="VPLink link pager-link next" href="/docs/storage.html" data-v-a5908b0f><!--[--><span class="desc" data-v-a5908b0f>Next page</span><span class="title" data-v-a5908b0f>Storage</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
-    <script>window.__VP_HASH_MAP__=JSON.parse("{\"ab.md\":\"hdxsvX5M\",\"building-with-agents.md\":\"PeZaZA1P\",\"concepts.md\":\"CqOsxbMU\",\"deployment.md\":\"Cdp4rTE4\",\"evals.md\":\"qeoGgPw0\",\"example-agents_approval-buddy.md\":\"CIiZ9coo\",\"example-agents_benny.md\":\"l7JTmm8X\",\"example-agents_bugbot.md\":\"Dp5JqHSQ\",\"example-agents_codebase-wiki.md\":\"D-lteFf0\",\"example-agents_codeowners-review.md\":\"BU2ZXLf-\",\"example-agents_concierge.md\":\"DA2al_NK\",\"example-agents_fsd.md\":\"DPz9ezO4\",\"example-agents_index.md\":\"xmJ87d_3\",\"example-agents_knowledge-base.md\":\"IneynQSR\",\"example-agents_oncall.md\":\"ZE0n6ZFN\",\"example-agents_security-reviewer.md\":\"Dkf1gyo6\",\"example-agents_slack-agent.md\":\"06jQXTAI\",\"example-agents_weather-agent.md\":\"DC3lINjo\",\"guides_agent-to-agent.md\":\"Bmbxy-FA\",\"guides_cloud-runtime.md\":\"V5igN4Sq\",\"guides_github.md\":\"B8QKtIpw\",\"guides_human-in-the-loop.md\":\"BWvT7UqY\",\"guides_mcp-oauth.md\":\"C7G7IykG\",\"guides_opentelemetry.md\":\"Csn7ZI25\",\"guides_slack.md\":\"zriQpU_9\",\"guides_webhooks.md\":\"DB-r_er9\",\"hillclimbing.md\":\"yXqdlv2R\",\"index.md\":\"COeoSkcF\",\"quickstart.md\":\"DZxBu44y\",\"reference_agent-config.md\":\"DHkkvcEZ\",\"reference_artifacts.md\":\"BGG4bZo-\",\"reference_channels.md\":\"MN8iDy8V\",\"reference_cli.md\":\"BDmjPPAm\",\"reference_connections.md\":\"CSQiNzre\",\"reference_hooks.md\":\"DJE5DXcT\",\"reference_http-api.md\":\"DGrw_wOu\",\"reference_instructions.md\":\"CgoV-YEb\",\"reference_playground.md\":\"CPZhfYaO\",\"reference_project-layout.md\":\"D3MdHM2z\",\"reference_prompt.md\":\"BaiweQxE\",\"reference_schedules.md\":\"gmfYzf_I\",\"reference_sessions.md\":\"DBVFi2Sx\",\"reference_skills.md\":\"B_jHN7JL\",\"reference_subagents.md\":\"DSrGLIuB\",\"reference_tools.md\":\"kJfJAP2m\",\"scaffolding-agents.md\":\"mkc3B_ZW\",\"storage.md\":\"CJYd5YmN\",\"templates_triage.md\":\"CConZt1z\",\"troubleshooting.md\":\"CkgwmVI0\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"Agent SDK\",\"description\":\"Filesystem-first framework for building and serving Cursor agents.\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"nav\":[{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Guides\",\"link\":\"/guides/webhooks\"},{\"text\":\"Examples\",\"link\":\"/example-agents/\"},{\"text\":\"Reference\",\"link\":\"/reference/project-layout\"}],\"search\":{\"provider\":\"local\"},\"outline\":{\"level\":[2,3]},\"sidebar\":[{\"text\":\"Getting started\",\"items\":[{\"text\":\"Overview\",\"link\":\"/\"},{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Scaffold an agent with Cursor\",\"link\":\"/scaffolding-agents\"},{\"text\":\"Concepts\",\"link\":\"/concepts\"}]},{\"text\":\"Templates\",\"items\":[{\"text\":\"Triage agent\",\"link\":\"/templates/triage\"}]},{\"text\":\"Improving agents\",\"items\":[{\"text\":\"Building agents with agents\",\"link\":\"/building-with-agents\"},{\"text\":\"Evals\",\"link\":\"/evals\"},{\"text\":\"Live A/B metrics\",\"link\":\"/ab\"},{\"text\":\"Storage\",\"link\":\"/storage\"},{\"text\":\"Hillclimbing\",\"link\":\"/hillclimbing\"}]},{\"text\":\"Guides\",\"items\":[{\"text\":\"Webhooks & custom channels\",\"link\":\"/guides/webhooks\"},{\"text\":\"GitHub\",\"link\":\"/guides/github\"},{\"text\":\"Slack\",\"link\":\"/guides/slack\"},{\"text\":\"Human-in-the-loop approvals\",\"link\":\"/guides/human-in-the-loop\"},{\"text\":\"Agent-to-agent\",\"link\":\"/guides/agent-to-agent\"},{\"text\":\"Host MCP OAuth\",\"link\":\"/guides/mcp-oauth\"},{\"text\":\"Cloud runtime\",\"link\":\"/guides/cloud-runtime\"},{\"text\":\"OpenTelemetry\",\"link\":\"/guides/opentelemetry\"}]},{\"text\":\"Example agents\",\"items\":[{\"text\":\"Choose an example\",\"link\":\"/example-agents/\"},{\"text\":\"Weather agent\",\"link\":\"/example-agents/weather-agent\"},{\"text\":\"Slack agent\",\"link\":\"/example-agents/slack-agent\"},{\"text\":\"Concierge\",\"link\":\"/example-agents/concierge\"},{\"text\":\"Playbook router\",\"link\":\"/example-agents/benny\"},{\"text\":\"Alert investigator\",\"link\":\"/example-agents/oncall\"},{\"text\":\"PR evidence reviewer\",\"link\":\"/example-agents/bugbot\"},{\"text\":\"Approval Buddy\",\"link\":\"/example-agents/approval-buddy\"},{\"text\":\"Security Reviewer\",\"link\":\"/example-agents/security-reviewer\"},{\"text\":\"Remote PR coordinator\",\"link\":\"/example-agents/fsd\"},{\"text\":\"Knowledge base\",\"link\":\"/example-agents/knowledge-base\"},{\"text\":\"Codebase wiki\",\"link\":\"/example-agents/codebase-wiki\"},{\"text\":\"Codeowners review\",\"link\":\"/example-agents/codeowners-review\"}]},{\"text\":\"Operating\",\"items\":[{\"text\":\"Deployment\",\"link\":\"/deployment\"},{\"text\":\"Fix common problems\",\"link\":\"/troubleshooting\"}]},{\"text\":\"Reference\",\"collapsed\":false,\"items\":[{\"text\":\"Project layout\",\"link\":\"/reference/project-layout\"},{\"text\":\"Agent config\",\"link\":\"/reference/agent-config\"},{\"text\":\"Instructions\",\"link\":\"/reference/instructions\"},{\"text\":\"Tools\",\"link\":\"/reference/tools\"},{\"text\":\"prompt\",\"link\":\"/reference/prompt\"},{\"text\":\"Skills\",\"link\":\"/reference/skills\"},{\"text\":\"MCP Connections\",\"link\":\"/reference/connections\"},{\"text\":\"Subagents\",\"link\":\"/reference/subagents\"},{\"text\":\"Channels\",\"link\":\"/reference/channels\"},{\"text\":\"Schedules & reminders\",\"link\":\"/reference/schedules\"},{\"text\":\"Hooks\",\"link\":\"/reference/hooks\"},{\"text\":\"Artifacts\",\"link\":\"/reference/artifacts\"},{\"text\":\"Sessions & streaming\",\"link\":\"/reference/sessions\"},{\"text\":\"Playground\",\"link\":\"/reference/playground\"},{\"text\":\"CLI\",\"link\":\"/reference/cli\"},{\"text\":\"HTTP API\",\"link\":\"/reference/http-api\"}]}]},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
+    <script>window.__VP_HASH_MAP__=JSON.parse("{\"ab.md\":\"DYjwREAP\",\"building-with-agents.md\":\"PeZaZA1P\",\"concepts.md\":\"CqOsxbMU\",\"deployment.md\":\"Cdp4rTE4\",\"evals.md\":\"CVe_O75-\",\"example-agents_approval-buddy.md\":\"CIiZ9coo\",\"example-agents_benny.md\":\"l7JTmm8X\",\"example-agents_bugbot.md\":\"Dp5JqHSQ\",\"example-agents_codebase-wiki.md\":\"D-lteFf0\",\"example-agents_codeowners-review.md\":\"BU2ZXLf-\",\"example-agents_concierge.md\":\"DA2al_NK\",\"example-agents_fsd.md\":\"DPz9ezO4\",\"example-agents_index.md\":\"xmJ87d_3\",\"example-agents_knowledge-base.md\":\"IneynQSR\",\"example-agents_oncall.md\":\"ZE0n6ZFN\",\"example-agents_security-reviewer.md\":\"Dkf1gyo6\",\"example-agents_slack-agent.md\":\"06jQXTAI\",\"example-agents_weather-agent.md\":\"DC3lINjo\",\"guides_agent-to-agent.md\":\"Bmbxy-FA\",\"guides_cloud-runtime.md\":\"V5igN4Sq\",\"guides_github.md\":\"B8QKtIpw\",\"guides_human-in-the-loop.md\":\"BWvT7UqY\",\"guides_mcp-oauth.md\":\"C7G7IykG\",\"guides_opentelemetry.md\":\"Csn7ZI25\",\"guides_slack.md\":\"zriQpU_9\",\"guides_webhooks.md\":\"DB-r_er9\",\"hillclimbing.md\":\"yXqdlv2R\",\"index.md\":\"COeoSkcF\",\"quickstart.md\":\"DZxBu44y\",\"reference_agent-config.md\":\"DHkkvcEZ\",\"reference_artifacts.md\":\"BGG4bZo-\",\"reference_channels.md\":\"MN8iDy8V\",\"reference_cli.md\":\"BDmjPPAm\",\"reference_connections.md\":\"CSQiNzre\",\"reference_hooks.md\":\"DJE5DXcT\",\"reference_http-api.md\":\"DMbdFGVQ\",\"reference_instructions.md\":\"CgoV-YEb\",\"reference_playground.md\":\"CPZhfYaO\",\"reference_project-layout.md\":\"D3MdHM2z\",\"reference_prompt.md\":\"BaiweQxE\",\"reference_schedules.md\":\"gmfYzf_I\",\"reference_sessions.md\":\"DBVFi2Sx\",\"reference_skills.md\":\"B_jHN7JL\",\"reference_subagents.md\":\"DSrGLIuB\",\"reference_tools.md\":\"kJfJAP2m\",\"scaffolding-agents.md\":\"mkc3B_ZW\",\"storage.md\":\"D8NQ389l\",\"templates_triage.md\":\"CConZt1z\",\"troubleshooting.md\":\"CkgwmVI0\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"Agent SDK\",\"description\":\"Filesystem-first framework for building and serving Cursor agents.\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"nav\":[{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Guides\",\"link\":\"/guides/webhooks\"},{\"text\":\"Examples\",\"link\":\"/example-agents/\"},{\"text\":\"Reference\",\"link\":\"/reference/project-layout\"}],\"search\":{\"provider\":\"local\"},\"outline\":{\"level\":[2,3]},\"sidebar\":[{\"text\":\"Getting started\",\"items\":[{\"text\":\"Overview\",\"link\":\"/\"},{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Scaffold an agent with Cursor\",\"link\":\"/scaffolding-agents\"},{\"text\":\"Concepts\",\"link\":\"/concepts\"}]},{\"text\":\"Templates\",\"items\":[{\"text\":\"Triage agent\",\"link\":\"/templates/triage\"}]},{\"text\":\"Improving agents\",\"items\":[{\"text\":\"Building agents with agents\",\"link\":\"/building-with-agents\"},{\"text\":\"Evals\",\"link\":\"/evals\"},{\"text\":\"Live A/B metrics\",\"link\":\"/ab\"},{\"text\":\"Storage\",\"link\":\"/storage\"},{\"text\":\"Hillclimbing\",\"link\":\"/hillclimbing\"}]},{\"text\":\"Guides\",\"items\":[{\"text\":\"Webhooks & custom channels\",\"link\":\"/guides/webhooks\"},{\"text\":\"GitHub\",\"link\":\"/guides/github\"},{\"text\":\"Slack\",\"link\":\"/guides/slack\"},{\"text\":\"Human-in-the-loop approvals\",\"link\":\"/guides/human-in-the-loop\"},{\"text\":\"Agent-to-agent\",\"link\":\"/guides/agent-to-agent\"},{\"text\":\"Host MCP OAuth\",\"link\":\"/guides/mcp-oauth\"},{\"text\":\"Cloud runtime\",\"link\":\"/guides/cloud-runtime\"},{\"text\":\"OpenTelemetry\",\"link\":\"/guides/opentelemetry\"}]},{\"text\":\"Example agents\",\"items\":[{\"text\":\"Choose an example\",\"link\":\"/example-agents/\"},{\"text\":\"Weather agent\",\"link\":\"/example-agents/weather-agent\"},{\"text\":\"Slack agent\",\"link\":\"/example-agents/slack-agent\"},{\"text\":\"Concierge\",\"link\":\"/example-agents/concierge\"},{\"text\":\"Playbook router\",\"link\":\"/example-agents/benny\"},{\"text\":\"Alert investigator\",\"link\":\"/example-agents/oncall\"},{\"text\":\"PR evidence reviewer\",\"link\":\"/example-agents/bugbot\"},{\"text\":\"Approval Buddy\",\"link\":\"/example-agents/approval-buddy\"},{\"text\":\"Security Reviewer\",\"link\":\"/example-agents/security-reviewer\"},{\"text\":\"Remote PR coordinator\",\"link\":\"/example-agents/fsd\"},{\"text\":\"Knowledge base\",\"link\":\"/example-agents/knowledge-base\"},{\"text\":\"Codebase wiki\",\"link\":\"/example-agents/codebase-wiki\"},{\"text\":\"Codeowners review\",\"link\":\"/example-agents/codeowners-review\"}]},{\"text\":\"Operating\",\"items\":[{\"text\":\"Deployment\",\"link\":\"/deployment\"},{\"text\":\"Fix common problems\",\"link\":\"/troubleshooting\"}]},{\"text\":\"Reference\",\"collapsed\":false,\"items\":[{\"text\":\"Project layout\",\"link\":\"/reference/project-layout\"},{\"text\":\"Agent config\",\"link\":\"/reference/agent-config\"},{\"text\":\"Instructions\",\"link\":\"/reference/instructions\"},{\"text\":\"Tools\",\"link\":\"/reference/tools\"},{\"text\":\"prompt\",\"link\":\"/reference/prompt\"},{\"text\":\"Skills\",\"link\":\"/reference/skills\"},{\"text\":\"MCP Connections\",\"link\":\"/reference/connections\"},{\"text\":\"Subagents\",\"link\":\"/reference/subagents\"},{\"text\":\"Channels\",\"link\":\"/reference/channels\"},{\"text\":\"Schedules & reminders\",\"link\":\"/reference/schedules\"},{\"text\":\"Hooks\",\"link\":\"/reference/hooks\"},{\"text\":\"Artifacts\",\"link\":\"/reference/artifacts\"},{\"text\":\"Sessions & streaming\",\"link\":\"/reference/sessions\"},{\"text\":\"Playground\",\"link\":\"/reference/playground\"},{\"text\":\"CLI\",\"link\":\"/reference/cli\"},{\"text\":\"HTTP API\",\"link\":\"/reference/http-api\"}]}]},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
     
   </body>
 </html>

package/dist/docs/assets/{ab.md.hdxsvX5M.js → ab.md.DYjwREAP.js}

@@ -1,4 +1,4 @@
-import{_ as s,c as a,o as t,ag as i}from"./chunks/framework.CAZyNGu9.js";const k=JSON.parse('{"title":"Live A/B metrics","description":"Assign sticky variants with defineAB, collect live performance metrics, and inspect per-arm results.","frontmatter":{"title":"Live A/B metrics","description":"Assign sticky variants with defineAB, collect live performance metrics, and inspect per-arm results."},"headers":[],"relativePath":"ab.md","filePath":"ab.md"}'),n={name:"ab.md"};function o(l,e,r,d,h,p){return t(),a("div",null,[...e[0]||(e[0]=[i(`<h1 id="live-a-b-metrics" tabindex="-1">Live A/B metrics <a class="header-anchor" href="#live-a-b-metrics" aria-label="Permalink to &quot;Live A/B metrics&quot;">​</a></h1><p>Use <code>defineAB</code> to compare variants on live agent sessions. New sessions receive a sticky assignment in each enrolled experiment. The Agent SDK folds their durable event streams into tool, token, failure, and wall-time metrics. You can send cumulative samples to your metrics backend and inspect aggregates in the playground.</p><p><code>defineAB</code> compares live variants through sticky assignment, instruction overlays, optional tool branches, and cumulative metrics. Metric callbacks observe the result without approving, rejecting, or failing a turn. Use <a href="./evals.html">evals</a> for pass/fail regression checks on fixed inputs.</p><div class="note custom-block github-alert"><p class="custom-block-title">NOTE</p><p>Import paths here use <code>@cursor/july/ab</code>. On projects still using <code>@anysphere/agent-serve</code>, swap the import. See <a href="/docs/#run-the-cli">Run the CLI</a> for the full rename table.</p></div><h2 id="choose-live-a-b-metrics-or-evals" tabindex="-1">Choose live A/B metrics or evals <a class="header-anchor" href="#choose-live-a-b-metrics-or-evals" aria-label="Permalink to &quot;Choose live A/B metrics or evals&quot;">​</a></h2><p>Both features read the session event stream, but they answer different questions.</p><table tabindex="0"><thead><tr><th></th><th>Live A/B metrics</th><th>Evals</th></tr></thead><tbody><tr><td>Question</td><td>How do variants compare on live sessions?</td><td>Does the agent still meet a fixed contract?</td></tr><tr><td>Location</td><td><code>agent/ab.ts</code> or <code>agent/ab/&lt;name&gt;.ts</code></td><td><code>evals/**/*.eval.ts</code></td></tr><tr><td>Input</td><td>Dev or production traffic</td><td>Frozen prompts and fixtures</td></tr><tr><td>Output</td><td>Cumulative metrics by session and arm</td><td>Pass/fail assertions</td></tr><tr><td>How it runs</td><td>Automatically on new live sessions</td><td><code>agent-sdk eval</code></td></tr></tbody></table><p>There is no <code>agent-sdk ab</code> command or assertion API.</p><h2 id="define-an-experiment" tabindex="-1">Define an experiment <a class="header-anchor" href="#define-an-experiment" aria-label="Permalink to &quot;Define an experiment&quot;">​</a></h2><p>Author one experiment in <code>agent/ab.ts</code>, add more under <code>agent/ab/&lt;name&gt;.ts</code>, or use both forms. Each file defines one experiment. The experiment name comes from <code>name</code> when set. Otherwise, The Agent SDK uses <code>ab</code> for <code>agent/ab.ts</code> and the file stem for files under <code>agent/ab/</code>.</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// agent/ab/concise-weather.ts</span></span>
+import{_ as s,c as t,o as a,ag as i}from"./chunks/framework.CAZyNGu9.js";const k=JSON.parse('{"title":"Live A/B metrics","description":"Assign sticky variants with defineAB, collect live performance metrics, and inspect per-arm results.","frontmatter":{"title":"Live A/B metrics","description":"Assign sticky variants with defineAB, collect live performance metrics, and inspect per-arm results."},"headers":[],"relativePath":"ab.md","filePath":"ab.md"}'),n={name:"ab.md"};function o(l,e,r,d,h,p){return a(),t("div",null,[...e[0]||(e[0]=[i(`<h1 id="live-a-b-metrics" tabindex="-1">Live A/B metrics <a class="header-anchor" href="#live-a-b-metrics" aria-label="Permalink to &quot;Live A/B metrics&quot;">​</a></h1><p>Use <code>defineAB</code> to compare variants on live agent sessions. New sessions receive a sticky assignment in each enrolled experiment. The Agent SDK folds their durable event streams into tool, token, failure, and wall-time metrics. You can send cumulative samples to your metrics backend and inspect aggregates in the playground.</p><p><code>defineAB</code> compares live variants through sticky assignment, instruction overlays, optional tool branches, and cumulative metrics. Metric callbacks observe the result without approving, rejecting, or failing a turn. Use <a href="./evals.html">evals</a> for pass/fail regression checks on fixed inputs.</p><div class="note custom-block github-alert"><p class="custom-block-title">NOTE</p><p>Import paths here use <code>@cursor/july/ab</code>. On projects still using <code>@anysphere/agent-serve</code>, swap the import. See <a href="/docs/#run-the-cli">Run the CLI</a> for the full rename table.</p></div><h2 id="choose-live-a-b-metrics-or-evals" tabindex="-1">Choose live A/B metrics or evals <a class="header-anchor" href="#choose-live-a-b-metrics-or-evals" aria-label="Permalink to &quot;Choose live A/B metrics or evals&quot;">​</a></h2><p>Both features read the session event stream, but they answer different questions.</p><table tabindex="0"><thead><tr><th></th><th>Live A/B metrics</th><th>Evals</th></tr></thead><tbody><tr><td>Question</td><td>How do variants compare on live sessions?</td><td>Does the agent still meet a fixed contract?</td></tr><tr><td>Location</td><td><code>agent/ab.ts</code> or <code>agent/ab/&lt;name&gt;.ts</code></td><td><code>evals/**/*.eval.ts</code></td></tr><tr><td>Input</td><td>Dev or production traffic</td><td>Frozen prompts and fixtures</td></tr><tr><td>Output</td><td>Cumulative metrics by session and arm</td><td>Pass/fail assertions</td></tr><tr><td>How it runs</td><td>Automatically on new live sessions</td><td><code>agent-sdk eval</code></td></tr></tbody></table><p>There is no <code>agent-sdk ab</code> command or assertion API.</p><h2 id="define-an-experiment" tabindex="-1">Define an experiment <a class="header-anchor" href="#define-an-experiment" aria-label="Permalink to &quot;Define an experiment&quot;">​</a></h2><p>Author one experiment in <code>agent/ab.ts</code>, add more under <code>agent/ab/&lt;name&gt;.ts</code>, or use both forms. Each file defines one experiment. The experiment name comes from <code>name</code> when set. Otherwise, The Agent SDK uses <code>ab</code> for <code>agent/ab.ts</code> and the file stem for files under <code>agent/ab/</code>.</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// agent/ab/concise-weather.ts</span></span>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> {</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  defineAB,</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  splitBySessionHash,</span></span>
@@ -46,7 +46,7 @@ import{_ as s,c as a,o as t,ag as i}from"./chunks/framework.CAZyNGu9.js";const k
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">  return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> conciseWeatherResult;</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span>
 <span class="line"></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> baselineWeatherResult;</span></span></code></pre></div><p>This makes the assignment available to deterministic code as well as the model prompt. Use both patterns together when one experiment must steer the prompt and host code at once.</p><p><code>defineAB</code> does not select a different model or runtime for each arm. Keep those settings in <code>agent/agent.ts</code>, or write explicit host logic when your experiment needs another behavior lever.</p><p>The split and selected arm can affect agent behavior. <code>derive</code> and <code>onSample</code> only observe the resulting event stream. Errors in either callback are logged and never fail the turn.</p><h2 id="collect-built-in-and-custom-metrics" tabindex="-1">Collect built-in and custom metrics <a class="header-anchor" href="#collect-built-in-and-custom-metrics" aria-label="Permalink to &quot;Collect built-in and custom metrics&quot;">​</a></h2><p>Metrics accumulate for each session and experiment. When one session joins several experiments, every enrolled experiment folds the same turn and tool events into its own counters.</p><table tabindex="0"><thead><tr><th>Metric</th><th>How the Agent SDK calculates it</th></tr></thead><tbody><tr><td><code>turns</code></td><td>Adds one on <code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>turnFailures</code></td><td>Adds one on <code>turn.failed</code></td></tr><tr><td><code>toolCalls</code></td><td>Adds one for each <code>action.result</code></td></tr><tr><td><code>toolErrors</code></td><td>Adds one when <code>action.result.data.isError</code> is true</td></tr><tr><td><code>inputTokens</code>, <code>outputTokens</code></td><td>Adds usage from completed turns</td></tr><tr><td><code>cacheReadTokens</code>, <code>cacheWriteTokens</code></td><td>Adds cache usage from completed turns</td></tr><tr><td><code>costUsd</code></td><td>Sums the estimated turn cost recorded on <code>turn.completed</code> (turns whose model has no known rates contribute 0)</td></tr><tr><td><code>wallTimeMs</code></td><td>Sums the time from <code>turn.started</code> to its completed or failed event</td></tr><tr><td><code>custom</code></td><td>Sums finite numeric deltas returned by <code>derive</code></td></tr></tbody></table><p><code>onSample</code> fires after every <code>turn.completed</code> and <code>turn.failed</code> event for an enrolled arm. The sample contains:</p><table tabindex="0"><thead><tr><th>Field</th><th>Value</th></tr></thead><tbody><tr><td><code>experiment</code></td><td>Experiment name</td></tr><tr><td><code>variant</code>, <code>variantLabel?</code></td><td>Sticky arm and optional display label</td></tr><tr><td><code>sessionId</code>, <code>channelId</code></td><td>Source session</td></tr><tr><td><code>metrics</code></td><td>Cumulative metrics through this turn</td></tr><tr><td><code>reason</code></td><td><code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>at</code></td><td>Terminal event timestamp</td></tr></tbody></table><p>The metrics are cumulative, not per-turn deltas. A second sample from the same session includes the first turn&#39;s counts.</p><p>Each <code>derive</code> extractor runs on every session event for its enrolled experiment, including streamed <code>message.appended</code> events. Keep it synchronous and cheap. Return a finite number to add a delta, or <code>null</code> to skip the event. Send samples to your metrics service from <code>onSample</code>; do not perform network or disk work in <code>derive</code>.</p><p>Skipped sessions never call <code>onSample</code>. Errors from <code>derive</code> or <code>onSample</code> are logged, then metric collection continues.</p><h2 id="inspect-assignments-and-results" tabindex="-1">Inspect assignments and results <a class="header-anchor" href="#inspect-assignments-and-results" aria-label="Permalink to &quot;Inspect assignments and results&quot;">​</a></h2><p>Open the playground&#39;s <strong>A/Bs</strong> tab to see aggregate arm totals and per-session assignments. The tab reads <code>GET /v1/abs</code>.</p><p>The response has two views of the same durable data:</p><table tabindex="0"><thead><tr><th>Field</th><th>Contents</th></tr></thead><tbody><tr><td><code>experiments</code></td><td>Declared variants, skipped-session count, arm session counts, and aggregate metrics</td></tr><tr><td><code>sessions</code></td><td>Visible sessions with their assignments and cumulative metrics</td></tr></tbody></table><p><code>GET /v1/abs</code> returns sessions visible to the current principal by default. In <code>--dev</code>, loopback requests include every session. Add <code>--allow-anonymous</code> to include every session from non-loopback callers too. This include-all behavior can still apply to <code>GET /v1/abs</code> in dev when bearer or custom auth keeps <code>GET /v1/sessions</code> owner-scoped.</p><p>Session <code>events.ndjson</code> is the source of truth for assignment + fold. <code>GET /v1/abs</code> recomputes aggregates from those logs. For durable sample and snapshot export, declare an <code>abs</code> table in <code>agent/storage.ts</code> — same idea as the eval-runs table. See <a href="./storage.html#eval-and-a-b-tables">Storage</a>.</p><h2 id="configure-the-playground-fold-window" tabindex="-1">Configure the playground fold window <a class="header-anchor" href="#configure-the-playground-fold-window" aria-label="Permalink to &quot;Configure the playground fold window&quot;">​</a></h2><p>Assignments and foldable metrics already persist in each session&#39;s <code>events.ndjson</code> under <code>--state-root</code>. The optional <code>agent/ab.config.ts</code> only caps how many sessions the playground and <code>GET /v1/abs</code> fold:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineABConfig } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/ab&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> baselineWeatherResult;</span></span></code></pre></div><p>This makes the assignment available to deterministic code as well as the model prompt. Use both patterns together when one experiment must steer the prompt and host code at once.</p><p><code>defineAB</code> does not select a different model or runtime for each arm. Keep those settings in <code>agent/agent.ts</code>, or write explicit host logic when your experiment needs another behavior lever.</p><p>The split and selected arm can affect agent behavior. <code>derive</code> and <code>onSample</code> only observe the resulting event stream. Errors in either callback are logged and never fail the turn.</p><h2 id="collect-built-in-and-custom-metrics" tabindex="-1">Collect built-in and custom metrics <a class="header-anchor" href="#collect-built-in-and-custom-metrics" aria-label="Permalink to &quot;Collect built-in and custom metrics&quot;">​</a></h2><p>Metrics accumulate for each session and experiment. When one session joins several experiments, every enrolled experiment folds the same turn and tool events into its own counters.</p><table tabindex="0"><thead><tr><th>Metric</th><th>How the Agent SDK calculates it</th></tr></thead><tbody><tr><td><code>turns</code></td><td>Adds one on <code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>turnFailures</code></td><td>Adds one on <code>turn.failed</code></td></tr><tr><td><code>toolCalls</code></td><td>Adds one for each <code>action.result</code></td></tr><tr><td><code>toolErrors</code></td><td>Adds one when <code>action.result.data.isError</code> is true</td></tr><tr><td><code>inputTokens</code>, <code>outputTokens</code></td><td>Adds usage from completed turns</td></tr><tr><td><code>cacheReadTokens</code>, <code>cacheWriteTokens</code></td><td>Adds cache usage from completed turns</td></tr><tr><td><code>costUsd</code></td><td>Sums the estimated turn cost recorded on <code>turn.completed</code> (turns whose model has no known rates contribute 0)</td></tr><tr><td><code>wallTimeMs</code></td><td>Sums the time from <code>turn.started</code> to its completed or failed event</td></tr><tr><td><code>custom</code></td><td>Sums finite numeric deltas returned by <code>derive</code></td></tr></tbody></table><p><code>onSample</code> fires after every <code>turn.completed</code> and <code>turn.failed</code> event for an enrolled arm. The sample contains:</p><table tabindex="0"><thead><tr><th>Field</th><th>Value</th></tr></thead><tbody><tr><td><code>experiment</code></td><td>Experiment name</td></tr><tr><td><code>variant</code>, <code>variantLabel?</code></td><td>Sticky arm and optional display label</td></tr><tr><td><code>sessionId</code>, <code>channelId</code></td><td>Source session</td></tr><tr><td><code>metrics</code></td><td>Cumulative metrics through this turn</td></tr><tr><td><code>reason</code></td><td><code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>at</code></td><td>Terminal event timestamp</td></tr></tbody></table><p>The metrics are cumulative, not per-turn deltas. A second sample from the same session includes the first turn&#39;s counts.</p><p>Each <code>derive</code> extractor runs on every session event for its enrolled experiment, including streamed <code>message.appended</code> events. Keep it synchronous and cheap. Return a finite number to add a delta, or <code>null</code> to skip the event. Send samples to your metrics service from <code>onSample</code>; do not perform network or disk work in <code>derive</code>.</p><p>Skipped sessions never call <code>onSample</code>. Errors from <code>derive</code> or <code>onSample</code> are logged, then metric collection continues.</p><h2 id="inspect-assignments-and-results" tabindex="-1">Inspect assignments and results <a class="header-anchor" href="#inspect-assignments-and-results" aria-label="Permalink to &quot;Inspect assignments and results&quot;">​</a></h2><p>Open the playground&#39;s <strong>A/Bs</strong> tab to see aggregate arm totals and per-session assignments. The tab reads <code>GET /v1/abs</code>.</p><p>The response has two views of the same durable data:</p><table tabindex="0"><thead><tr><th>Field</th><th>Contents</th></tr></thead><tbody><tr><td><code>experiments</code></td><td>Declared variants, skipped-session count, arm session counts, and aggregate metrics</td></tr><tr><td><code>sessions</code></td><td>Visible sessions with their assignments and cumulative metrics</td></tr></tbody></table><p><code>GET /v1/abs</code> returns sessions visible to the current principal by default. In <code>--dev</code>, loopback requests include every session. Add <code>--allow-anonymous</code> to include every session from non-loopback callers too. This include-all behavior can still apply to <code>GET /v1/abs</code> in dev when bearer or custom auth keeps <code>GET /v1/sessions</code> owner-scoped.</p><p>Session <code>events.ndjson</code> is the source of truth for assignment + fold. <code>GET /v1/abs</code> recomputes aggregates from those logs. Any <code>agent/storage.ts</code> exports samples and snapshots durably: an authored <code>abs</code> table when the backend has a native shape for it, or the table derived over the KV core otherwise. See <a href="./storage.html#eval-and-a-b-tables">Storage</a>.</p><h2 id="configure-the-playground-fold-window" tabindex="-1">Configure the playground fold window <a class="header-anchor" href="#configure-the-playground-fold-window" aria-label="Permalink to &quot;Configure the playground fold window&quot;">​</a></h2><p>Assignments and foldable metrics already persist in each session&#39;s <code>events.ndjson</code> under <code>--state-root</code>. The optional <code>agent/ab.config.ts</code> only caps how many sessions the playground and <code>GET /v1/abs</code> fold:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineABConfig } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;@cursor/july/ab&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
 <span class="line"></span>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineABConfig</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
 <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">  // Optional — defaults to 200. Only affects GET /v1/abs / A/Bs tab.</span></span>

package/dist/docs/assets/{ab.md.hdxsvX5M.lean.js → ab.md.DYjwREAP.lean.js}

@@ -1 +1 @@
-import{_ as s,c as a,o as t,ag as i}from"./chunks/framework.CAZyNGu9.js";const k=JSON.parse('{"title":"Live A/B metrics","description":"Assign sticky variants with defineAB, collect live performance metrics, and inspect per-arm results.","frontmatter":{"title":"Live A/B metrics","description":"Assign sticky variants with defineAB, collect live performance metrics, and inspect per-arm results."},"headers":[],"relativePath":"ab.md","filePath":"ab.md"}'),n={name:"ab.md"};function o(l,e,r,d,h,p){return t(),a("div",null,[...e[0]||(e[0]=[i("",68)])])}const u=s(n,[["render",o]]);export{k as __pageData,u as default};
+import{_ as s,c as t,o as a,ag as i}from"./chunks/framework.CAZyNGu9.js";const k=JSON.parse('{"title":"Live A/B metrics","description":"Assign sticky variants with defineAB, collect live performance metrics, and inspect per-arm results.","frontmatter":{"title":"Live A/B metrics","description":"Assign sticky variants with defineAB, collect live performance metrics, and inspect per-arm results."},"headers":[],"relativePath":"ab.md","filePath":"ab.md"}'),n={name:"ab.md"};function o(l,e,r,d,h,p){return a(),t("div",null,[...e[0]||(e[0]=[i("",68)])])}const u=s(n,[["render",o]]);export{k as __pageData,u as default};

package/dist/docs/assets/{app.IEXEim5m.js → app.BpWAgjaR.js}

@@ -1 +1 @@
-import{t as p}from"./chunks/theme.DroGdz_Q.js";import{R as s,a2 as i,a3 as u,a4 as c,a5 as l,a6 as f,a7 as d,a8 as m,a9 as h,aa as g,ab as A,d as v,u as y,v as C,s as P,ac as b,ad as w,ae as R,af as E}from"./chunks/framework.CAZyNGu9.js";function r(e){if(e.extends){const a=r(e.extends);return{...a,...e,async enhanceApp(t){a.enhanceApp&&await a.enhanceApp(t),e.enhanceApp&&await e.enhanceApp(t)}}}return e}const n=r(p),S=v({name:"VitePressApp",setup(){const{site:e,lang:a,dir:t}=y();return C(()=>{P(()=>{document.documentElement.lang=a.value,document.documentElement.dir=t.value})}),e.value.router.prefetchLinks&&b(),w(),R(),n.setup&&n.setup(),()=>E(n.Layout)}});async function T(){globalThis.__VITEPRESS__=!0;const e=_(),a=D();a.provide(u,e);const t=c(e.route);return a.provide(l,t),a.component("Content",f),a.component("ClientOnly",d),Object.defineProperties(a.config.globalProperties,{$frontmatter:{get(){return t.frontmatter.value}},$params:{get(){return t.page.value.params}}}),n.enhanceApp&&await n.enhanceApp({app:a,router:e,siteData:m}),{app:a,router:e,data:t}}function D(){return A(S)}function _(){let e=s;return h(a=>{let t=g(a),o=null;return t&&(e&&(t=t.replace(/\.js$/,".lean.js")),o=import(t)),s&&(e=!1),o},n.NotFound)}s&&T().then(({app:e,router:a,data:t})=>{a.go().then(()=>{i(a.route,t.site),e.mount("#app")})});export{T as createApp};
+import{t as p}from"./chunks/theme.B5IvUxOB.js";import{R as s,a2 as i,a3 as u,a4 as c,a5 as l,a6 as f,a7 as d,a8 as m,a9 as h,aa as g,ab as A,d as v,u as y,v as C,s as P,ac as b,ad as w,ae as R,af as E}from"./chunks/framework.CAZyNGu9.js";function r(e){if(e.extends){const a=r(e.extends);return{...a,...e,async enhanceApp(t){a.enhanceApp&&await a.enhanceApp(t),e.enhanceApp&&await e.enhanceApp(t)}}}return e}const n=r(p),S=v({name:"VitePressApp",setup(){const{site:e,lang:a,dir:t}=y();return C(()=>{P(()=>{document.documentElement.lang=a.value,document.documentElement.dir=t.value})}),e.value.router.prefetchLinks&&b(),w(),R(),n.setup&&n.setup(),()=>E(n.Layout)}});async function T(){globalThis.__VITEPRESS__=!0;const e=_(),a=D();a.provide(u,e);const t=c(e.route);return a.provide(l,t),a.component("Content",f),a.component("ClientOnly",d),Object.defineProperties(a.config.globalProperties,{$frontmatter:{get(){return t.frontmatter.value}},$params:{get(){return t.page.value.params}}}),n.enhanceApp&&await n.enhanceApp({app:a,router:e,siteData:m}),{app:a,router:e,data:t}}function D(){return A(S)}function _(){let e=s;return h(a=>{let t=g(a),o=null;return t&&(e&&(t=t.replace(/\.js$/,".lean.js")),o=import(t)),s&&(e=!1),o},n.NotFound)}s&&T().then(({app:e,router:a,data:t})=>{a.go().then(()=>{i(a.route,t.site),e.mount("#app")})});export{T as createApp};