@cursor/july 0.1.48 → 0.1.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/docs/404.html +2 -2
- package/dist/docs/ab.html +5 -5
- package/dist/docs/assets/{ab.md.hdxsvX5M.js → ab.md.DYjwREAP.js} +2 -2
- package/dist/docs/assets/{ab.md.hdxsvX5M.lean.js → ab.md.DYjwREAP.lean.js} +1 -1
- package/dist/docs/assets/{app.BC1P20xI.js → app.BpWAgjaR.js} +1 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.D00d772K.js +1 -0
- package/dist/docs/assets/chunks/{VPLocalSearchBox.C_900xn2.js → VPLocalSearchBox.CnAktZtG.js} +1 -1
- package/dist/docs/assets/chunks/{theme.CT135Jhh.js → theme.B5IvUxOB.js} +2 -2
- package/dist/docs/assets/{evals.md.qeoGgPw0.js → evals.md.CVe_O75-.js} +3 -3
- package/dist/docs/assets/{evals.md.qeoGgPw0.lean.js → evals.md.CVe_O75-.lean.js} +1 -1
- package/dist/docs/assets/{reference_http-api.md.DGrw_wOu.js → reference_http-api.md.DMbdFGVQ.js} +1 -1
- package/dist/docs/assets/{storage.md.CJYd5YmN.js → storage.md.D8NQ389l.js} +3 -3
- package/dist/docs/assets/storage.md.D8NQ389l.lean.js +1 -0
- package/dist/docs/building-with-agents.html +3 -3
- package/dist/docs/concepts.html +3 -3
- package/dist/docs/deployment.html +3 -3
- package/dist/docs/evals.html +6 -6
- package/dist/docs/example-agents/approval-buddy.html +3 -3
- package/dist/docs/example-agents/benny.html +3 -3
- package/dist/docs/example-agents/bugbot.html +3 -3
- package/dist/docs/example-agents/codebase-wiki.html +3 -3
- package/dist/docs/example-agents/codeowners-review.html +3 -3
- package/dist/docs/example-agents/concierge.html +3 -3
- package/dist/docs/example-agents/fsd.html +3 -3
- package/dist/docs/example-agents/index.html +3 -3
- package/dist/docs/example-agents/knowledge-base.html +3 -3
- package/dist/docs/example-agents/oncall.html +3 -3
- package/dist/docs/example-agents/security-reviewer.html +3 -3
- package/dist/docs/example-agents/slack-agent.html +3 -3
- package/dist/docs/example-agents/weather-agent.html +3 -3
- package/dist/docs/guides/agent-to-agent.html +3 -3
- package/dist/docs/guides/cloud-runtime.html +3 -3
- package/dist/docs/guides/github.html +3 -3
- package/dist/docs/guides/human-in-the-loop.html +3 -3
- package/dist/docs/guides/mcp-oauth.html +3 -3
- package/dist/docs/guides/opentelemetry.html +3 -3
- package/dist/docs/guides/slack.html +3 -3
- package/dist/docs/guides/webhooks.html +3 -3
- package/dist/docs/hashmap.json +1 -1
- package/dist/docs/hillclimbing.html +3 -3
- package/dist/docs/index.html +3 -3
- package/dist/docs/quickstart.html +3 -3
- package/dist/docs/reference/agent-config.html +3 -3
- package/dist/docs/reference/artifacts.html +3 -3
- package/dist/docs/reference/channels.html +3 -3
- package/dist/docs/reference/cli.html +3 -3
- package/dist/docs/reference/connections.html +3 -3
- package/dist/docs/reference/hooks.html +3 -3
- package/dist/docs/reference/http-api.html +5 -5
- package/dist/docs/reference/instructions.html +3 -3
- package/dist/docs/reference/playground.html +3 -3
- package/dist/docs/reference/project-layout.html +3 -3
- package/dist/docs/reference/prompt.html +3 -3
- package/dist/docs/reference/schedules.html +3 -3
- package/dist/docs/reference/sessions.html +3 -3
- package/dist/docs/reference/skills.html +3 -3
- package/dist/docs/reference/subagents.html +3 -3
- package/dist/docs/reference/tools.html +3 -3
- package/dist/docs/scaffolding-agents.html +3 -3
- package/dist/docs/storage.html +5 -5
- package/dist/docs/templates/triage.html +3 -3
- package/dist/docs/troubleshooting.html +3 -3
- package/dist/internal/cli-slack.js +0 -3
- package/dist/internal/event-log.d.ts +17 -0
- package/dist/internal/event-log.d.ts.map +1 -1
- package/dist/internal/event-log.js +32 -2
- package/dist/internal/process-memory-telemetry.d.ts +90 -0
- package/dist/internal/process-memory-telemetry.d.ts.map +1 -0
- package/dist/internal/process-memory-telemetry.js +259 -0
- package/dist/internal/server.d.ts.map +1 -1
- package/dist/internal/server.js +9 -10
- package/dist/internal/session-engine.d.ts +4 -0
- package/dist/internal/session-engine.d.ts.map +1 -1
- package/dist/internal/session-engine.js +19 -5
- package/dist/internal/slack-provision-client.d.ts +4 -4
- package/dist/internal/slack-provision-client.js +7 -7
- package/dist/playground/assets/index-Dox74yWJ.css +1 -0
- package/dist/playground/assets/{index-DEVTyeOu.js → index-Dq_HqyU-.js} +41 -41
- package/dist/playground/index.html +2 -2
- package/dist/storage.d.ts +36 -10
- package/dist/storage.d.ts.map +1 -1
- package/dist/storage.js +82 -6
- package/docs/ab.md +4 -3
- package/docs/evals.md +10 -7
- package/docs/reference/http-api.md +4 -3
- package/docs/storage.md +38 -22
- package/package.json +1 -1
- package/skills/setup-slack/SKILL.md +1 -1
- package/src/internal/cli-slack.ts +0 -5
- package/src/internal/event-log.ts +45 -7
- package/src/internal/process-memory-telemetry.ts +373 -0
- package/src/internal/server.ts +13 -0
- package/src/internal/session-engine.ts +20 -1
- package/src/internal/slack-provision-client.ts +7 -7
- package/src/storage.ts +130 -11
- package/dist/docs/assets/chunks/@localSearchIndexroot.DV5SAHnQ.js +0 -1
- package/dist/docs/assets/storage.md.CJYd5YmN.lean.js +0 -1
- package/dist/playground/assets/index-CiCmH6R-.css +0 -1
- /package/dist/docs/assets/{reference_http-api.md.DGrw_wOu.lean.js → reference_http-api.md.DMbdFGVQ.lean.js} +0 -0
package/dist/docs/404.html
CHANGED
|
@@ -9,14 +9,14 @@
|
|
|
9
9
|
<link rel="preload stylesheet" href="/docs/assets/style.CK3hwt6q.css" as="style">
|
|
10
10
|
<link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
|
|
11
11
|
|
|
12
|
-
<script type="module" src="/docs/assets/app.
|
|
12
|
+
<script type="module" src="/docs/assets/app.BpWAgjaR.js"></script>
|
|
13
13
|
<link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
|
|
14
14
|
<script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
|
|
15
15
|
<script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
|
|
16
16
|
</head>
|
|
17
17
|
<body>
|
|
18
18
|
<div id="app"></div>
|
|
19
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"ab.md\":\"
|
|
19
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"ab.md\":\"DYjwREAP\",\"building-with-agents.md\":\"PeZaZA1P\",\"concepts.md\":\"CqOsxbMU\",\"deployment.md\":\"Cdp4rTE4\",\"evals.md\":\"CVe_O75-\",\"example-agents_approval-buddy.md\":\"CIiZ9coo\",\"example-agents_benny.md\":\"l7JTmm8X\",\"example-agents_bugbot.md\":\"Dp5JqHSQ\",\"example-agents_codebase-wiki.md\":\"D-lteFf0\",\"example-agents_codeowners-review.md\":\"BU2ZXLf-\",\"example-agents_concierge.md\":\"DA2al_NK\",\"example-agents_fsd.md\":\"DPz9ezO4\",\"example-agents_index.md\":\"xmJ87d_3\",\"example-agents_knowledge-base.md\":\"IneynQSR\",\"example-agents_oncall.md\":\"ZE0n6ZFN\",\"example-agents_security-reviewer.md\":\"Dkf1gyo6\",\"example-agents_slack-agent.md\":\"06jQXTAI\",\"example-agents_weather-agent.md\":\"DC3lINjo\",\"guides_agent-to-agent.md\":\"Bmbxy-FA\",\"guides_cloud-runtime.md\":\"V5igN4Sq\",\"guides_github.md\":\"B8QKtIpw\",\"guides_human-in-the-loop.md\":\"BWvT7UqY\",\"guides_mcp-oauth.md\":\"C7G7IykG\",\"guides_opentelemetry.md\":\"Csn7ZI25\",\"guides_slack.md\":\"zriQpU_9\",\"guides_webhooks.md\":\"DB-r_er9\",\"hillclimbing.md\":\"yXqdlv2R\",\"index.md\":\"COeoSkcF\",\"quickstart.md\":\"DZxBu44y\",\"reference_agent-config.md\":\"DHkkvcEZ\",\"reference_artifacts.md\":\"BGG4bZo-\",\"reference_channels.md\":\"MN8iDy8V\",\"reference_cli.md\":\"BDmjPPAm\",\"reference_connections.md\":\"CSQiNzre\",\"reference_hooks.md\":\"DJE5DXcT\",\"reference_http-api.md\":\"DMbdFGVQ\",\"reference_instructions.md\":\"CgoV-YEb\",\"reference_playground.md\":\"CPZhfYaO\",\"reference_project-layout.md\":\"D3MdHM2z\",\"reference_prompt.md\":\"BaiweQxE\",\"reference_schedules.md\":\"gmfYzf_I\",\"reference_sessions.md\":\"DBVFi2Sx\",\"reference_skills.md\":\"B_jHN7JL\",\"reference_subagents.md\":\"DSrGLIuB\",\"reference_tools.md\":\"kJfJAP2m\",\"scaffolding-agents.md\":\"mkc3B_ZW\",\"storage.md\":\"D8NQ389l\",\"templates_triage.md\":\"CConZt1z\",\"troubleshooting.md\":\"CkgwmVI0\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"Agent SDK\",\"description\":\"Filesystem-first framework for building and serving Cursor agents.\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"nav\":[{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Guides\",\"link\":\"/guides/webhooks\"},{\"text\":\"Examples\",\"link\":\"/example-agents/\"},{\"text\":\"Reference\",\"link\":\"/reference/project-layout\"}],\"search\":{\"provider\":\"local\"},\"outline\":{\"level\":[2,3]},\"sidebar\":[{\"text\":\"Getting started\",\"items\":[{\"text\":\"Overview\",\"link\":\"/\"},{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Scaffold an agent with Cursor\",\"link\":\"/scaffolding-agents\"},{\"text\":\"Concepts\",\"link\":\"/concepts\"}]},{\"text\":\"Templates\",\"items\":[{\"text\":\"Triage agent\",\"link\":\"/templates/triage\"}]},{\"text\":\"Improving agents\",\"items\":[{\"text\":\"Building agents with agents\",\"link\":\"/building-with-agents\"},{\"text\":\"Evals\",\"link\":\"/evals\"},{\"text\":\"Live A/B metrics\",\"link\":\"/ab\"},{\"text\":\"Storage\",\"link\":\"/storage\"},{\"text\":\"Hillclimbing\",\"link\":\"/hillclimbing\"}]},{\"text\":\"Guides\",\"items\":[{\"text\":\"Webhooks & custom channels\",\"link\":\"/guides/webhooks\"},{\"text\":\"GitHub\",\"link\":\"/guides/github\"},{\"text\":\"Slack\",\"link\":\"/guides/slack\"},{\"text\":\"Human-in-the-loop approvals\",\"link\":\"/guides/human-in-the-loop\"},{\"text\":\"Agent-to-agent\",\"link\":\"/guides/agent-to-agent\"},{\"text\":\"Host MCP OAuth\",\"link\":\"/guides/mcp-oauth\"},{\"text\":\"Cloud runtime\",\"link\":\"/guides/cloud-runtime\"},{\"text\":\"OpenTelemetry\",\"link\":\"/guides/opentelemetry\"}]},{\"text\":\"Example agents\",\"items\":[{\"text\":\"Choose an example\",\"link\":\"/example-agents/\"},{\"text\":\"Weather agent\",\"link\":\"/example-agents/weather-agent\"},{\"text\":\"Slack agent\",\"link\":\"/example-agents/slack-agent\"},{\"text\":\"Concierge\",\"link\":\"/example-agents/concierge\"},{\"text\":\"Playbook router\",\"link\":\"/example-agents/benny\"},{\"text\":\"Alert investigator\",\"link\":\"/example-agents/oncall\"},{\"text\":\"PR evidence reviewer\",\"link\":\"/example-agents/bugbot\"},{\"text\":\"Approval Buddy\",\"link\":\"/example-agents/approval-buddy\"},{\"text\":\"Security Reviewer\",\"link\":\"/example-agents/security-reviewer\"},{\"text\":\"Remote PR coordinator\",\"link\":\"/example-agents/fsd\"},{\"text\":\"Knowledge base\",\"link\":\"/example-agents/knowledge-base\"},{\"text\":\"Codebase wiki\",\"link\":\"/example-agents/codebase-wiki\"},{\"text\":\"Codeowners review\",\"link\":\"/example-agents/codeowners-review\"}]},{\"text\":\"Operating\",\"items\":[{\"text\":\"Deployment\",\"link\":\"/deployment\"},{\"text\":\"Fix common problems\",\"link\":\"/troubleshooting\"}]},{\"text\":\"Reference\",\"collapsed\":false,\"items\":[{\"text\":\"Project layout\",\"link\":\"/reference/project-layout\"},{\"text\":\"Agent config\",\"link\":\"/reference/agent-config\"},{\"text\":\"Instructions\",\"link\":\"/reference/instructions\"},{\"text\":\"Tools\",\"link\":\"/reference/tools\"},{\"text\":\"prompt\",\"link\":\"/reference/prompt\"},{\"text\":\"Skills\",\"link\":\"/reference/skills\"},{\"text\":\"MCP Connections\",\"link\":\"/reference/connections\"},{\"text\":\"Subagents\",\"link\":\"/reference/subagents\"},{\"text\":\"Channels\",\"link\":\"/reference/channels\"},{\"text\":\"Schedules & reminders\",\"link\":\"/reference/schedules\"},{\"text\":\"Hooks\",\"link\":\"/reference/hooks\"},{\"text\":\"Artifacts\",\"link\":\"/reference/artifacts\"},{\"text\":\"Sessions & streaming\",\"link\":\"/reference/sessions\"},{\"text\":\"Playground\",\"link\":\"/reference/playground\"},{\"text\":\"CLI\",\"link\":\"/reference/cli\"},{\"text\":\"HTTP API\",\"link\":\"/reference/http-api\"}]}]},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
|
|
20
20
|
|
|
21
21
|
</body>
|
|
22
22
|
</html>
|
package/dist/docs/ab.html
CHANGED
|
@@ -9,11 +9,11 @@
|
|
|
9
9
|
<link rel="preload stylesheet" href="/docs/assets/style.CK3hwt6q.css" as="style">
|
|
10
10
|
<link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
|
|
11
11
|
|
|
12
|
-
<script type="module" src="/docs/assets/app.
|
|
12
|
+
<script type="module" src="/docs/assets/app.BpWAgjaR.js"></script>
|
|
13
13
|
<link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
|
|
14
|
-
<link rel="modulepreload" href="/docs/assets/chunks/theme.
|
|
14
|
+
<link rel="modulepreload" href="/docs/assets/chunks/theme.B5IvUxOB.js">
|
|
15
15
|
<link rel="modulepreload" href="/docs/assets/chunks/framework.CAZyNGu9.js">
|
|
16
|
-
<link rel="modulepreload" href="/docs/assets/ab.md.
|
|
16
|
+
<link rel="modulepreload" href="/docs/assets/ab.md.DYjwREAP.lean.js">
|
|
17
17
|
<script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
|
|
18
18
|
<script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
|
|
19
19
|
</head>
|
|
@@ -66,13 +66,13 @@
|
|
|
66
66
|
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> conciseWeatherResult;</span></span>
|
|
67
67
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span>
|
|
68
68
|
<span class="line"></span>
|
|
69
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> baselineWeatherResult;</span></span></code></pre></div><p>This makes the assignment available to deterministic code as well as the model prompt. Use both patterns together when one experiment must steer the prompt and host code at once.</p><p><code>defineAB</code> does not select a different model or runtime for each arm. Keep those settings in <code>agent/agent.ts</code>, or write explicit host logic when your experiment needs another behavior lever.</p><p>The split and selected arm can affect agent behavior. <code>derive</code> and <code>onSample</code> only observe the resulting event stream. Errors in either callback are logged and never fail the turn.</p><h2 id="collect-built-in-and-custom-metrics" tabindex="-1">Collect built-in and custom metrics <a class="header-anchor" href="#collect-built-in-and-custom-metrics" aria-label="Permalink to "Collect built-in and custom metrics""></a></h2><p>Metrics accumulate for each session and experiment. When one session joins several experiments, every enrolled experiment folds the same turn and tool events into its own counters.</p><table tabindex="0"><thead><tr><th>Metric</th><th>How the Agent SDK calculates it</th></tr></thead><tbody><tr><td><code>turns</code></td><td>Adds one on <code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>turnFailures</code></td><td>Adds one on <code>turn.failed</code></td></tr><tr><td><code>toolCalls</code></td><td>Adds one for each <code>action.result</code></td></tr><tr><td><code>toolErrors</code></td><td>Adds one when <code>action.result.data.isError</code> is true</td></tr><tr><td><code>inputTokens</code>, <code>outputTokens</code></td><td>Adds usage from completed turns</td></tr><tr><td><code>cacheReadTokens</code>, <code>cacheWriteTokens</code></td><td>Adds cache usage from completed turns</td></tr><tr><td><code>costUsd</code></td><td>Sums the estimated turn cost recorded on <code>turn.completed</code> (turns whose model has no known rates contribute 0)</td></tr><tr><td><code>wallTimeMs</code></td><td>Sums the time from <code>turn.started</code> to its completed or failed event</td></tr><tr><td><code>custom</code></td><td>Sums finite numeric deltas returned by <code>derive</code></td></tr></tbody></table><p><code>onSample</code> fires after every <code>turn.completed</code> and <code>turn.failed</code> event for an enrolled arm. The sample contains:</p><table tabindex="0"><thead><tr><th>Field</th><th>Value</th></tr></thead><tbody><tr><td><code>experiment</code></td><td>Experiment name</td></tr><tr><td><code>variant</code>, <code>variantLabel?</code></td><td>Sticky arm and optional display label</td></tr><tr><td><code>sessionId</code>, <code>channelId</code></td><td>Source session</td></tr><tr><td><code>metrics</code></td><td>Cumulative metrics through this turn</td></tr><tr><td><code>reason</code></td><td><code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>at</code></td><td>Terminal event timestamp</td></tr></tbody></table><p>The metrics are cumulative, not per-turn deltas. A second sample from the same session includes the first turn's counts.</p><p>Each <code>derive</code> extractor runs on every session event for its enrolled experiment, including streamed <code>message.appended</code> events. Keep it synchronous and cheap. Return a finite number to add a delta, or <code>null</code> to skip the event. Send samples to your metrics service from <code>onSample</code>; do not perform network or disk work in <code>derive</code>.</p><p>Skipped sessions never call <code>onSample</code>. Errors from <code>derive</code> or <code>onSample</code> are logged, then metric collection continues.</p><h2 id="inspect-assignments-and-results" tabindex="-1">Inspect assignments and results <a class="header-anchor" href="#inspect-assignments-and-results" aria-label="Permalink to "Inspect assignments and results""></a></h2><p>Open the playground's <strong>A/Bs</strong> tab to see aggregate arm totals and per-session assignments. The tab reads <code>GET /v1/abs</code>.</p><p>The response has two views of the same durable data:</p><table tabindex="0"><thead><tr><th>Field</th><th>Contents</th></tr></thead><tbody><tr><td><code>experiments</code></td><td>Declared variants, skipped-session count, arm session counts, and aggregate metrics</td></tr><tr><td><code>sessions</code></td><td>Visible sessions with their assignments and cumulative metrics</td></tr></tbody></table><p><code>GET /v1/abs</code> returns sessions visible to the current principal by default. In <code>--dev</code>, loopback requests include every session. Add <code>--allow-anonymous</code> to include every session from non-loopback callers too. This include-all behavior can still apply to <code>GET /v1/abs</code> in dev when bearer or custom auth keeps <code>GET /v1/sessions</code> owner-scoped.</p><p>Session <code>events.ndjson</code> is the source of truth for assignment + fold. <code>GET /v1/abs</code> recomputes aggregates from those logs.
|
|
69
|
+
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> baselineWeatherResult;</span></span></code></pre></div><p>This makes the assignment available to deterministic code as well as the model prompt. Use both patterns together when one experiment must steer the prompt and host code at once.</p><p><code>defineAB</code> does not select a different model or runtime for each arm. Keep those settings in <code>agent/agent.ts</code>, or write explicit host logic when your experiment needs another behavior lever.</p><p>The split and selected arm can affect agent behavior. <code>derive</code> and <code>onSample</code> only observe the resulting event stream. Errors in either callback are logged and never fail the turn.</p><h2 id="collect-built-in-and-custom-metrics" tabindex="-1">Collect built-in and custom metrics <a class="header-anchor" href="#collect-built-in-and-custom-metrics" aria-label="Permalink to "Collect built-in and custom metrics""></a></h2><p>Metrics accumulate for each session and experiment. When one session joins several experiments, every enrolled experiment folds the same turn and tool events into its own counters.</p><table tabindex="0"><thead><tr><th>Metric</th><th>How the Agent SDK calculates it</th></tr></thead><tbody><tr><td><code>turns</code></td><td>Adds one on <code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>turnFailures</code></td><td>Adds one on <code>turn.failed</code></td></tr><tr><td><code>toolCalls</code></td><td>Adds one for each <code>action.result</code></td></tr><tr><td><code>toolErrors</code></td><td>Adds one when <code>action.result.data.isError</code> is true</td></tr><tr><td><code>inputTokens</code>, <code>outputTokens</code></td><td>Adds usage from completed turns</td></tr><tr><td><code>cacheReadTokens</code>, <code>cacheWriteTokens</code></td><td>Adds cache usage from completed turns</td></tr><tr><td><code>costUsd</code></td><td>Sums the estimated turn cost recorded on <code>turn.completed</code> (turns whose model has no known rates contribute 0)</td></tr><tr><td><code>wallTimeMs</code></td><td>Sums the time from <code>turn.started</code> to its completed or failed event</td></tr><tr><td><code>custom</code></td><td>Sums finite numeric deltas returned by <code>derive</code></td></tr></tbody></table><p><code>onSample</code> fires after every <code>turn.completed</code> and <code>turn.failed</code> event for an enrolled arm. The sample contains:</p><table tabindex="0"><thead><tr><th>Field</th><th>Value</th></tr></thead><tbody><tr><td><code>experiment</code></td><td>Experiment name</td></tr><tr><td><code>variant</code>, <code>variantLabel?</code></td><td>Sticky arm and optional display label</td></tr><tr><td><code>sessionId</code>, <code>channelId</code></td><td>Source session</td></tr><tr><td><code>metrics</code></td><td>Cumulative metrics through this turn</td></tr><tr><td><code>reason</code></td><td><code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>at</code></td><td>Terminal event timestamp</td></tr></tbody></table><p>The metrics are cumulative, not per-turn deltas. A second sample from the same session includes the first turn's counts.</p><p>Each <code>derive</code> extractor runs on every session event for its enrolled experiment, including streamed <code>message.appended</code> events. Keep it synchronous and cheap. Return a finite number to add a delta, or <code>null</code> to skip the event. Send samples to your metrics service from <code>onSample</code>; do not perform network or disk work in <code>derive</code>.</p><p>Skipped sessions never call <code>onSample</code>. Errors from <code>derive</code> or <code>onSample</code> are logged, then metric collection continues.</p><h2 id="inspect-assignments-and-results" tabindex="-1">Inspect assignments and results <a class="header-anchor" href="#inspect-assignments-and-results" aria-label="Permalink to "Inspect assignments and results""></a></h2><p>Open the playground's <strong>A/Bs</strong> tab to see aggregate arm totals and per-session assignments. The tab reads <code>GET /v1/abs</code>.</p><p>The response has two views of the same durable data:</p><table tabindex="0"><thead><tr><th>Field</th><th>Contents</th></tr></thead><tbody><tr><td><code>experiments</code></td><td>Declared variants, skipped-session count, arm session counts, and aggregate metrics</td></tr><tr><td><code>sessions</code></td><td>Visible sessions with their assignments and cumulative metrics</td></tr></tbody></table><p><code>GET /v1/abs</code> returns sessions visible to the current principal by default. In <code>--dev</code>, loopback requests include every session. Add <code>--allow-anonymous</code> to include every session from non-loopback callers too. This include-all behavior can still apply to <code>GET /v1/abs</code> in dev when bearer or custom auth keeps <code>GET /v1/sessions</code> owner-scoped.</p><p>Session <code>events.ndjson</code> is the source of truth for assignment + fold. <code>GET /v1/abs</code> recomputes aggregates from those logs. Any <code>agent/storage.ts</code> exports samples and snapshots durably: an authored <code>abs</code> table when the backend has a native shape for it, or the table derived over the KV core otherwise. See <a href="./storage.html#eval-and-a-b-tables">Storage</a>.</p><h2 id="configure-the-playground-fold-window" tabindex="-1">Configure the playground fold window <a class="header-anchor" href="#configure-the-playground-fold-window" aria-label="Permalink to "Configure the playground fold window""></a></h2><p>Assignments and foldable metrics already persist in each session's <code>events.ndjson</code> under <code>--state-root</code>. The optional <code>agent/ab.config.ts</code> only caps how many sessions the playground and <code>GET /v1/abs</code> fold:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineABConfig } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "@cursor/july/ab"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
|
|
70
70
|
<span class="line"></span>
|
|
71
71
|
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineABConfig</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
|
|
72
72
|
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> // Optional — defaults to 200. Only affects GET /v1/abs / A/Bs tab.</span></span>
|
|
73
73
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> maxPlaygroundSessions: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">500</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
74
74
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p><code>maxPlaygroundSessions</code> keeps the newest sessions in the fold. It does not prune session logs or change assignment. For export to S3, a DB, or your metrics vendor, send samples from <code>onSample</code> or declare a storage <code>abs</code> table.</p><h2 id="keep-assignments-durable" tabindex="-1">Keep assignments durable <a class="header-anchor" href="#keep-assignments-durable" aria-label="Permalink to "Keep assignments durable""></a></h2><p>The append-only <code>events.ndjson</code> stream is the source of truth. Each <code>ab.assigned</code> event persists a variant key or null skip. Built-in metrics come from the turn and tool events that follow it.</p><p>After a server restart or a parked session resumes, the live collector replays the stream to rebuild cumulative counters. Replay does not call <code>onSample</code> (or write to the storage <code>abs</code> table) for historical turns. Only a new completed or failed turn emits another sample.</p><p>The snapshot API also replays <code>derive</code> across the full stream, so custom totals match the current extractor. Changing a derive function can change historical snapshot totals. Treat metric definitions as versioned experiment code.</p><h2 id="keep-eval-traffic-separate" tabindex="-1">Keep eval traffic separate <a class="header-anchor" href="#keep-eval-traffic-separate" aria-label="Permalink to "Keep eval traffic separate""></a></h2><p>Sessions created by <code>agent-sdk eval</code> and the playground Evals runner use <code>purpose: "eval"</code>. They skip A/B enrollment entirely:</p><ul><li>No split function runs.</li><li>No <code>ab.assigned</code> event is recorded.</li><li>No <code>onSample</code> callback fires.</li><li>The session is omitted from <code>GET /v1/abs</code>.</li></ul><p>Ordinary chat, <code>agent-sdk run</code>, Slack, GitHub, and other channel sessions use the live purpose. You do not need <code>splitIf</code> to exclude eval traffic.</p><h2 id="know-the-boundaries" tabindex="-1">Know the boundaries <a class="header-anchor" href="#know-the-boundaries" aria-label="Permalink to "Know the boundaries""></a></h2><p><code>defineAB</code> provides sticky assignment, variant instructions, <code>session.abs</code> for tools, cumulative metrics, and local inspection. It does not provide:</p><ul><li>A test command, assertion API, or pass/fail result</li><li>Statistical significance calculations</li><li>An experiment rollout or lifecycle service</li><li>Per-variant model or runtime configuration</li><li>A built-in analytics warehouse (bring your own via <code>onSample</code> or the storage <code>abs</code> table)</li></ul><p>Use <a href="./evals.html">evals</a> to protect known behavior. Use <code>onSample</code> or a storage <code>abs</code> table when you need sample/snapshot exports beyond the session event log.</p><h2 id="what-s-next" tabindex="-1">What's next <a class="header-anchor" href="#what-s-next" aria-label="Permalink to "What's next""></a></h2><p>Continue with these pages:</p><ul><li><a href="./evals.html">Evals</a>: pass/fail regression checks on fixed inputs</li><li><a href="./hillclimbing.html">Hillclimbing</a>: improve an agent against fixed fixtures</li><li><a href="./reference/hooks.html">Hooks</a>: other event-stream consumers</li><li><a href="./reference/sessions.html">Sessions and streaming</a>: the <code>ab.assigned</code> event and durable log</li><li><a href="./reference/playground.html">Playground</a>: the A/Bs tab</li><li><a href="./reference/http-api.html">HTTP API</a>: <code>GET /v1/abs</code></li><li><a href="./../skills/ab/SKILL.html">Live A/B metrics skill</a>: have a coding agent wire an experiment</li></ul></div></div></main><footer class="VPDocFooter" data-v-d5d7ccd1 data-v-a5908b0f><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-a5908b0f><span class="visually-hidden" id="doc-footer-aria-label" data-v-a5908b0f>Pager</span><div class="pager" data-v-a5908b0f><a class="VPLink link pager-link prev" href="/docs/evals.html" data-v-a5908b0f><!--[--><span class="desc" data-v-a5908b0f>Previous page</span><span class="title" data-v-a5908b0f>Evals</span><!--]--></a></div><div class="pager" data-v-a5908b0f><a class="VPLink link pager-link next" href="/docs/storage.html" data-v-a5908b0f><!--[--><span class="desc" data-v-a5908b0f>Next page</span><span class="title" data-v-a5908b0f>Storage</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
75
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"ab.md\":\"
|
|
75
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"ab.md\":\"DYjwREAP\",\"building-with-agents.md\":\"PeZaZA1P\",\"concepts.md\":\"CqOsxbMU\",\"deployment.md\":\"Cdp4rTE4\",\"evals.md\":\"CVe_O75-\",\"example-agents_approval-buddy.md\":\"CIiZ9coo\",\"example-agents_benny.md\":\"l7JTmm8X\",\"example-agents_bugbot.md\":\"Dp5JqHSQ\",\"example-agents_codebase-wiki.md\":\"D-lteFf0\",\"example-agents_codeowners-review.md\":\"BU2ZXLf-\",\"example-agents_concierge.md\":\"DA2al_NK\",\"example-agents_fsd.md\":\"DPz9ezO4\",\"example-agents_index.md\":\"xmJ87d_3\",\"example-agents_knowledge-base.md\":\"IneynQSR\",\"example-agents_oncall.md\":\"ZE0n6ZFN\",\"example-agents_security-reviewer.md\":\"Dkf1gyo6\",\"example-agents_slack-agent.md\":\"06jQXTAI\",\"example-agents_weather-agent.md\":\"DC3lINjo\",\"guides_agent-to-agent.md\":\"Bmbxy-FA\",\"guides_cloud-runtime.md\":\"V5igN4Sq\",\"guides_github.md\":\"B8QKtIpw\",\"guides_human-in-the-loop.md\":\"BWvT7UqY\",\"guides_mcp-oauth.md\":\"C7G7IykG\",\"guides_opentelemetry.md\":\"Csn7ZI25\",\"guides_slack.md\":\"zriQpU_9\",\"guides_webhooks.md\":\"DB-r_er9\",\"hillclimbing.md\":\"yXqdlv2R\",\"index.md\":\"COeoSkcF\",\"quickstart.md\":\"DZxBu44y\",\"reference_agent-config.md\":\"DHkkvcEZ\",\"reference_artifacts.md\":\"BGG4bZo-\",\"reference_channels.md\":\"MN8iDy8V\",\"reference_cli.md\":\"BDmjPPAm\",\"reference_connections.md\":\"CSQiNzre\",\"reference_hooks.md\":\"DJE5DXcT\",\"reference_http-api.md\":\"DMbdFGVQ\",\"reference_instructions.md\":\"CgoV-YEb\",\"reference_playground.md\":\"CPZhfYaO\",\"reference_project-layout.md\":\"D3MdHM2z\",\"reference_prompt.md\":\"BaiweQxE\",\"reference_schedules.md\":\"gmfYzf_I\",\"reference_sessions.md\":\"DBVFi2Sx\",\"reference_skills.md\":\"B_jHN7JL\",\"reference_subagents.md\":\"DSrGLIuB\",\"reference_tools.md\":\"kJfJAP2m\",\"scaffolding-agents.md\":\"mkc3B_ZW\",\"storage.md\":\"D8NQ389l\",\"templates_triage.md\":\"CConZt1z\",\"troubleshooting.md\":\"CkgwmVI0\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"Agent SDK\",\"description\":\"Filesystem-first framework for building and serving Cursor agents.\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"nav\":[{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Guides\",\"link\":\"/guides/webhooks\"},{\"text\":\"Examples\",\"link\":\"/example-agents/\"},{\"text\":\"Reference\",\"link\":\"/reference/project-layout\"}],\"search\":{\"provider\":\"local\"},\"outline\":{\"level\":[2,3]},\"sidebar\":[{\"text\":\"Getting started\",\"items\":[{\"text\":\"Overview\",\"link\":\"/\"},{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Scaffold an agent with Cursor\",\"link\":\"/scaffolding-agents\"},{\"text\":\"Concepts\",\"link\":\"/concepts\"}]},{\"text\":\"Templates\",\"items\":[{\"text\":\"Triage agent\",\"link\":\"/templates/triage\"}]},{\"text\":\"Improving agents\",\"items\":[{\"text\":\"Building agents with agents\",\"link\":\"/building-with-agents\"},{\"text\":\"Evals\",\"link\":\"/evals\"},{\"text\":\"Live A/B metrics\",\"link\":\"/ab\"},{\"text\":\"Storage\",\"link\":\"/storage\"},{\"text\":\"Hillclimbing\",\"link\":\"/hillclimbing\"}]},{\"text\":\"Guides\",\"items\":[{\"text\":\"Webhooks & custom channels\",\"link\":\"/guides/webhooks\"},{\"text\":\"GitHub\",\"link\":\"/guides/github\"},{\"text\":\"Slack\",\"link\":\"/guides/slack\"},{\"text\":\"Human-in-the-loop approvals\",\"link\":\"/guides/human-in-the-loop\"},{\"text\":\"Agent-to-agent\",\"link\":\"/guides/agent-to-agent\"},{\"text\":\"Host MCP OAuth\",\"link\":\"/guides/mcp-oauth\"},{\"text\":\"Cloud runtime\",\"link\":\"/guides/cloud-runtime\"},{\"text\":\"OpenTelemetry\",\"link\":\"/guides/opentelemetry\"}]},{\"text\":\"Example agents\",\"items\":[{\"text\":\"Choose an example\",\"link\":\"/example-agents/\"},{\"text\":\"Weather agent\",\"link\":\"/example-agents/weather-agent\"},{\"text\":\"Slack agent\",\"link\":\"/example-agents/slack-agent\"},{\"text\":\"Concierge\",\"link\":\"/example-agents/concierge\"},{\"text\":\"Playbook router\",\"link\":\"/example-agents/benny\"},{\"text\":\"Alert investigator\",\"link\":\"/example-agents/oncall\"},{\"text\":\"PR evidence reviewer\",\"link\":\"/example-agents/bugbot\"},{\"text\":\"Approval Buddy\",\"link\":\"/example-agents/approval-buddy\"},{\"text\":\"Security Reviewer\",\"link\":\"/example-agents/security-reviewer\"},{\"text\":\"Remote PR coordinator\",\"link\":\"/example-agents/fsd\"},{\"text\":\"Knowledge base\",\"link\":\"/example-agents/knowledge-base\"},{\"text\":\"Codebase wiki\",\"link\":\"/example-agents/codebase-wiki\"},{\"text\":\"Codeowners review\",\"link\":\"/example-agents/codeowners-review\"}]},{\"text\":\"Operating\",\"items\":[{\"text\":\"Deployment\",\"link\":\"/deployment\"},{\"text\":\"Fix common problems\",\"link\":\"/troubleshooting\"}]},{\"text\":\"Reference\",\"collapsed\":false,\"items\":[{\"text\":\"Project layout\",\"link\":\"/reference/project-layout\"},{\"text\":\"Agent config\",\"link\":\"/reference/agent-config\"},{\"text\":\"Instructions\",\"link\":\"/reference/instructions\"},{\"text\":\"Tools\",\"link\":\"/reference/tools\"},{\"text\":\"prompt\",\"link\":\"/reference/prompt\"},{\"text\":\"Skills\",\"link\":\"/reference/skills\"},{\"text\":\"MCP Connections\",\"link\":\"/reference/connections\"},{\"text\":\"Subagents\",\"link\":\"/reference/subagents\"},{\"text\":\"Channels\",\"link\":\"/reference/channels\"},{\"text\":\"Schedules & reminders\",\"link\":\"/reference/schedules\"},{\"text\":\"Hooks\",\"link\":\"/reference/hooks\"},{\"text\":\"Artifacts\",\"link\":\"/reference/artifacts\"},{\"text\":\"Sessions & streaming\",\"link\":\"/reference/sessions\"},{\"text\":\"Playground\",\"link\":\"/reference/playground\"},{\"text\":\"CLI\",\"link\":\"/reference/cli\"},{\"text\":\"HTTP API\",\"link\":\"/reference/http-api\"}]}]},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
|
|
76
76
|
|
|
77
77
|
</body>
|
|
78
78
|
</html>
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{_ as s,c as
|
|
1
|
+
import{_ as s,c as t,o as a,ag as i}from"./chunks/framework.CAZyNGu9.js";const k=JSON.parse('{"title":"Live A/B metrics","description":"Assign sticky variants with defineAB, collect live performance metrics, and inspect per-arm results.","frontmatter":{"title":"Live A/B metrics","description":"Assign sticky variants with defineAB, collect live performance metrics, and inspect per-arm results."},"headers":[],"relativePath":"ab.md","filePath":"ab.md"}'),n={name:"ab.md"};function o(l,e,r,d,h,p){return a(),t("div",null,[...e[0]||(e[0]=[i(`<h1 id="live-a-b-metrics" tabindex="-1">Live A/B metrics <a class="header-anchor" href="#live-a-b-metrics" aria-label="Permalink to "Live A/B metrics""></a></h1><p>Use <code>defineAB</code> to compare variants on live agent sessions. New sessions receive a sticky assignment in each enrolled experiment. The Agent SDK folds their durable event streams into tool, token, failure, and wall-time metrics. You can send cumulative samples to your metrics backend and inspect aggregates in the playground.</p><p><code>defineAB</code> compares live variants through sticky assignment, instruction overlays, optional tool branches, and cumulative metrics. Metric callbacks observe the result without approving, rejecting, or failing a turn. Use <a href="./evals.html">evals</a> for pass/fail regression checks on fixed inputs.</p><div class="note custom-block github-alert"><p class="custom-block-title">NOTE</p><p>Import paths here use <code>@cursor/july/ab</code>. On projects still using <code>@anysphere/agent-serve</code>, swap the import. See <a href="/docs/#run-the-cli">Run the CLI</a> for the full rename table.</p></div><h2 id="choose-live-a-b-metrics-or-evals" tabindex="-1">Choose live A/B metrics or evals <a class="header-anchor" href="#choose-live-a-b-metrics-or-evals" aria-label="Permalink to "Choose live A/B metrics or evals""></a></h2><p>Both features read the session event stream, but they answer different questions.</p><table tabindex="0"><thead><tr><th></th><th>Live A/B metrics</th><th>Evals</th></tr></thead><tbody><tr><td>Question</td><td>How do variants compare on live sessions?</td><td>Does the agent still meet a fixed contract?</td></tr><tr><td>Location</td><td><code>agent/ab.ts</code> or <code>agent/ab/<name>.ts</code></td><td><code>evals/**/*.eval.ts</code></td></tr><tr><td>Input</td><td>Dev or production traffic</td><td>Frozen prompts and fixtures</td></tr><tr><td>Output</td><td>Cumulative metrics by session and arm</td><td>Pass/fail assertions</td></tr><tr><td>How it runs</td><td>Automatically on new live sessions</td><td><code>agent-sdk eval</code></td></tr></tbody></table><p>There is no <code>agent-sdk ab</code> command or assertion API.</p><h2 id="define-an-experiment" tabindex="-1">Define an experiment <a class="header-anchor" href="#define-an-experiment" aria-label="Permalink to "Define an experiment""></a></h2><p>Author one experiment in <code>agent/ab.ts</code>, add more under <code>agent/ab/<name>.ts</code>, or use both forms. Each file defines one experiment. The experiment name comes from <code>name</code> when set. Otherwise, The Agent SDK uses <code>ab</code> for <code>agent/ab.ts</code> and the file stem for files under <code>agent/ab/</code>.</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// agent/ab/concise-weather.ts</span></span>
|
|
2
2
|
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> {</span></span>
|
|
3
3
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> defineAB,</span></span>
|
|
4
4
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> splitBySessionHash,</span></span>
|
|
@@ -46,7 +46,7 @@ import{_ as s,c as a,o as t,ag as i}from"./chunks/framework.CAZyNGu9.js";const k
|
|
|
46
46
|
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> conciseWeatherResult;</span></span>
|
|
47
47
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span>
|
|
48
48
|
<span class="line"></span>
|
|
49
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> baselineWeatherResult;</span></span></code></pre></div><p>This makes the assignment available to deterministic code as well as the model prompt. Use both patterns together when one experiment must steer the prompt and host code at once.</p><p><code>defineAB</code> does not select a different model or runtime for each arm. Keep those settings in <code>agent/agent.ts</code>, or write explicit host logic when your experiment needs another behavior lever.</p><p>The split and selected arm can affect agent behavior. <code>derive</code> and <code>onSample</code> only observe the resulting event stream. Errors in either callback are logged and never fail the turn.</p><h2 id="collect-built-in-and-custom-metrics" tabindex="-1">Collect built-in and custom metrics <a class="header-anchor" href="#collect-built-in-and-custom-metrics" aria-label="Permalink to "Collect built-in and custom metrics""></a></h2><p>Metrics accumulate for each session and experiment. When one session joins several experiments, every enrolled experiment folds the same turn and tool events into its own counters.</p><table tabindex="0"><thead><tr><th>Metric</th><th>How the Agent SDK calculates it</th></tr></thead><tbody><tr><td><code>turns</code></td><td>Adds one on <code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>turnFailures</code></td><td>Adds one on <code>turn.failed</code></td></tr><tr><td><code>toolCalls</code></td><td>Adds one for each <code>action.result</code></td></tr><tr><td><code>toolErrors</code></td><td>Adds one when <code>action.result.data.isError</code> is true</td></tr><tr><td><code>inputTokens</code>, <code>outputTokens</code></td><td>Adds usage from completed turns</td></tr><tr><td><code>cacheReadTokens</code>, <code>cacheWriteTokens</code></td><td>Adds cache usage from completed turns</td></tr><tr><td><code>costUsd</code></td><td>Sums the estimated turn cost recorded on <code>turn.completed</code> (turns whose model has no known rates contribute 0)</td></tr><tr><td><code>wallTimeMs</code></td><td>Sums the time from <code>turn.started</code> to its completed or failed event</td></tr><tr><td><code>custom</code></td><td>Sums finite numeric deltas returned by <code>derive</code></td></tr></tbody></table><p><code>onSample</code> fires after every <code>turn.completed</code> and <code>turn.failed</code> event for an enrolled arm. The sample contains:</p><table tabindex="0"><thead><tr><th>Field</th><th>Value</th></tr></thead><tbody><tr><td><code>experiment</code></td><td>Experiment name</td></tr><tr><td><code>variant</code>, <code>variantLabel?</code></td><td>Sticky arm and optional display label</td></tr><tr><td><code>sessionId</code>, <code>channelId</code></td><td>Source session</td></tr><tr><td><code>metrics</code></td><td>Cumulative metrics through this turn</td></tr><tr><td><code>reason</code></td><td><code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>at</code></td><td>Terminal event timestamp</td></tr></tbody></table><p>The metrics are cumulative, not per-turn deltas. A second sample from the same session includes the first turn's counts.</p><p>Each <code>derive</code> extractor runs on every session event for its enrolled experiment, including streamed <code>message.appended</code> events. Keep it synchronous and cheap. Return a finite number to add a delta, or <code>null</code> to skip the event. Send samples to your metrics service from <code>onSample</code>; do not perform network or disk work in <code>derive</code>.</p><p>Skipped sessions never call <code>onSample</code>. Errors from <code>derive</code> or <code>onSample</code> are logged, then metric collection continues.</p><h2 id="inspect-assignments-and-results" tabindex="-1">Inspect assignments and results <a class="header-anchor" href="#inspect-assignments-and-results" aria-label="Permalink to "Inspect assignments and results""></a></h2><p>Open the playground's <strong>A/Bs</strong> tab to see aggregate arm totals and per-session assignments. The tab reads <code>GET /v1/abs</code>.</p><p>The response has two views of the same durable data:</p><table tabindex="0"><thead><tr><th>Field</th><th>Contents</th></tr></thead><tbody><tr><td><code>experiments</code></td><td>Declared variants, skipped-session count, arm session counts, and aggregate metrics</td></tr><tr><td><code>sessions</code></td><td>Visible sessions with their assignments and cumulative metrics</td></tr></tbody></table><p><code>GET /v1/abs</code> returns sessions visible to the current principal by default. In <code>--dev</code>, loopback requests include every session. Add <code>--allow-anonymous</code> to include every session from non-loopback callers too. This include-all behavior can still apply to <code>GET /v1/abs</code> in dev when bearer or custom auth keeps <code>GET /v1/sessions</code> owner-scoped.</p><p>Session <code>events.ndjson</code> is the source of truth for assignment + fold. <code>GET /v1/abs</code> recomputes aggregates from those logs.
|
|
49
|
+
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> baselineWeatherResult;</span></span></code></pre></div><p>This makes the assignment available to deterministic code as well as the model prompt. Use both patterns together when one experiment must steer the prompt and host code at once.</p><p><code>defineAB</code> does not select a different model or runtime for each arm. Keep those settings in <code>agent/agent.ts</code>, or write explicit host logic when your experiment needs another behavior lever.</p><p>The split and selected arm can affect agent behavior. <code>derive</code> and <code>onSample</code> only observe the resulting event stream. Errors in either callback are logged and never fail the turn.</p><h2 id="collect-built-in-and-custom-metrics" tabindex="-1">Collect built-in and custom metrics <a class="header-anchor" href="#collect-built-in-and-custom-metrics" aria-label="Permalink to "Collect built-in and custom metrics""></a></h2><p>Metrics accumulate for each session and experiment. When one session joins several experiments, every enrolled experiment folds the same turn and tool events into its own counters.</p><table tabindex="0"><thead><tr><th>Metric</th><th>How the Agent SDK calculates it</th></tr></thead><tbody><tr><td><code>turns</code></td><td>Adds one on <code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>turnFailures</code></td><td>Adds one on <code>turn.failed</code></td></tr><tr><td><code>toolCalls</code></td><td>Adds one for each <code>action.result</code></td></tr><tr><td><code>toolErrors</code></td><td>Adds one when <code>action.result.data.isError</code> is true</td></tr><tr><td><code>inputTokens</code>, <code>outputTokens</code></td><td>Adds usage from completed turns</td></tr><tr><td><code>cacheReadTokens</code>, <code>cacheWriteTokens</code></td><td>Adds cache usage from completed turns</td></tr><tr><td><code>costUsd</code></td><td>Sums the estimated turn cost recorded on <code>turn.completed</code> (turns whose model has no known rates contribute 0)</td></tr><tr><td><code>wallTimeMs</code></td><td>Sums the time from <code>turn.started</code> to its completed or failed event</td></tr><tr><td><code>custom</code></td><td>Sums finite numeric deltas returned by <code>derive</code></td></tr></tbody></table><p><code>onSample</code> fires after every <code>turn.completed</code> and <code>turn.failed</code> event for an enrolled arm. The sample contains:</p><table tabindex="0"><thead><tr><th>Field</th><th>Value</th></tr></thead><tbody><tr><td><code>experiment</code></td><td>Experiment name</td></tr><tr><td><code>variant</code>, <code>variantLabel?</code></td><td>Sticky arm and optional display label</td></tr><tr><td><code>sessionId</code>, <code>channelId</code></td><td>Source session</td></tr><tr><td><code>metrics</code></td><td>Cumulative metrics through this turn</td></tr><tr><td><code>reason</code></td><td><code>turn.completed</code> or <code>turn.failed</code></td></tr><tr><td><code>at</code></td><td>Terminal event timestamp</td></tr></tbody></table><p>The metrics are cumulative, not per-turn deltas. A second sample from the same session includes the first turn's counts.</p><p>Each <code>derive</code> extractor runs on every session event for its enrolled experiment, including streamed <code>message.appended</code> events. Keep it synchronous and cheap. Return a finite number to add a delta, or <code>null</code> to skip the event. Send samples to your metrics service from <code>onSample</code>; do not perform network or disk work in <code>derive</code>.</p><p>Skipped sessions never call <code>onSample</code>. Errors from <code>derive</code> or <code>onSample</code> are logged, then metric collection continues.</p><h2 id="inspect-assignments-and-results" tabindex="-1">Inspect assignments and results <a class="header-anchor" href="#inspect-assignments-and-results" aria-label="Permalink to "Inspect assignments and results""></a></h2><p>Open the playground's <strong>A/Bs</strong> tab to see aggregate arm totals and per-session assignments. The tab reads <code>GET /v1/abs</code>.</p><p>The response has two views of the same durable data:</p><table tabindex="0"><thead><tr><th>Field</th><th>Contents</th></tr></thead><tbody><tr><td><code>experiments</code></td><td>Declared variants, skipped-session count, arm session counts, and aggregate metrics</td></tr><tr><td><code>sessions</code></td><td>Visible sessions with their assignments and cumulative metrics</td></tr></tbody></table><p><code>GET /v1/abs</code> returns sessions visible to the current principal by default. In <code>--dev</code>, loopback requests include every session. Add <code>--allow-anonymous</code> to include every session from non-loopback callers too. This include-all behavior can still apply to <code>GET /v1/abs</code> in dev when bearer or custom auth keeps <code>GET /v1/sessions</code> owner-scoped.</p><p>Session <code>events.ndjson</code> is the source of truth for assignment + fold. <code>GET /v1/abs</code> recomputes aggregates from those logs. Any <code>agent/storage.ts</code> exports samples and snapshots durably: an authored <code>abs</code> table when the backend has a native shape for it, or the table derived over the KV core otherwise. See <a href="./storage.html#eval-and-a-b-tables">Storage</a>.</p><h2 id="configure-the-playground-fold-window" tabindex="-1">Configure the playground fold window <a class="header-anchor" href="#configure-the-playground-fold-window" aria-label="Permalink to "Configure the playground fold window""></a></h2><p>Assignments and foldable metrics already persist in each session's <code>events.ndjson</code> under <code>--state-root</code>. The optional <code>agent/ab.config.ts</code> only caps how many sessions the playground and <code>GET /v1/abs</code> fold:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineABConfig } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "@cursor/july/ab"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
|
|
50
50
|
<span class="line"></span>
|
|
51
51
|
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineABConfig</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
|
|
52
52
|
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> // Optional — defaults to 200. Only affects GET /v1/abs / A/Bs tab.</span></span>
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{_ as s,c as
|
|
1
|
+
import{_ as s,c as t,o as a,ag as i}from"./chunks/framework.CAZyNGu9.js";const k=JSON.parse('{"title":"Live A/B metrics","description":"Assign sticky variants with defineAB, collect live performance metrics, and inspect per-arm results.","frontmatter":{"title":"Live A/B metrics","description":"Assign sticky variants with defineAB, collect live performance metrics, and inspect per-arm results."},"headers":[],"relativePath":"ab.md","filePath":"ab.md"}'),n={name:"ab.md"};function o(l,e,r,d,h,p){return a(),t("div",null,[...e[0]||(e[0]=[i("",68)])])}const u=s(n,[["render",o]]);export{k as __pageData,u as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{t as p}from"./chunks/theme.
|
|
1
|
+
import{t as p}from"./chunks/theme.B5IvUxOB.js";import{R as s,a2 as i,a3 as u,a4 as c,a5 as l,a6 as f,a7 as d,a8 as m,a9 as h,aa as g,ab as A,d as v,u as y,v as C,s as P,ac as b,ad as w,ae as R,af as E}from"./chunks/framework.CAZyNGu9.js";function r(e){if(e.extends){const a=r(e.extends);return{...a,...e,async enhanceApp(t){a.enhanceApp&&await a.enhanceApp(t),e.enhanceApp&&await e.enhanceApp(t)}}}return e}const n=r(p),S=v({name:"VitePressApp",setup(){const{site:e,lang:a,dir:t}=y();return C(()=>{P(()=>{document.documentElement.lang=a.value,document.documentElement.dir=t.value})}),e.value.router.prefetchLinks&&b(),w(),R(),n.setup&&n.setup(),()=>E(n.Layout)}});async function T(){globalThis.__VITEPRESS__=!0;const e=_(),a=D();a.provide(u,e);const t=c(e.route);return a.provide(l,t),a.component("Content",f),a.component("ClientOnly",d),Object.defineProperties(a.config.globalProperties,{$frontmatter:{get(){return t.frontmatter.value}},$params:{get(){return t.page.value.params}}}),n.enhanceApp&&await n.enhanceApp({app:a,router:e,siteData:m}),{app:a,router:e,data:t}}function D(){return A(S)}function _(){let e=s;return h(a=>{let t=g(a),o=null;return t&&(e&&(t=t.replace(/\.js$/,".lean.js")),o=import(t)),s&&(e=!1),o},n.NotFound)}s&&T().then(({app:e,router:a,data:t})=>{a.go().then(()=>{i(a.route,t.site),e.mount("#app")})});export{T as createApp};
|