@cursor/july 0.1.91 → 0.1.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -0
- package/dist/docs/404.html +3 -1
- package/dist/docs/ab.html +4 -2
- package/dist/docs/ab.md +338 -0
- package/dist/docs/building-with-agents.html +4 -2
- package/dist/docs/building-with-agents.md +124 -0
- package/dist/docs/concepts.html +4 -2
- package/dist/docs/concepts.md +174 -0
- package/dist/docs/deployment.html +4 -2
- package/dist/docs/deployment.md +464 -0
- package/dist/docs/evals.html +4 -2
- package/dist/docs/evals.md +481 -0
- package/dist/docs/example-agents/approval-buddy.html +4 -2
- package/dist/docs/example-agents/approval-buddy.md +266 -0
- package/dist/docs/example-agents/benny.html +4 -2
- package/dist/docs/example-agents/benny.md +182 -0
- package/dist/docs/example-agents/bugbot.html +4 -2
- package/dist/docs/example-agents/bugbot.md +229 -0
- package/dist/docs/example-agents/codebase-wiki.html +4 -2
- package/dist/docs/example-agents/codebase-wiki.md +170 -0
- package/dist/docs/example-agents/codeowners-review.html +4 -2
- package/dist/docs/example-agents/codeowners-review.md +192 -0
- package/dist/docs/example-agents/concierge.html +4 -2
- package/dist/docs/example-agents/concierge.md +201 -0
- package/dist/docs/example-agents/fsd.html +4 -2
- package/dist/docs/example-agents/fsd.md +329 -0
- package/dist/docs/example-agents/index.html +4 -2
- package/dist/docs/example-agents/index.md +102 -0
- package/dist/docs/example-agents/knowledge-base.html +4 -2
- package/dist/docs/example-agents/knowledge-base.md +168 -0
- package/dist/docs/example-agents/oncall.html +4 -2
- package/dist/docs/example-agents/oncall.md +212 -0
- package/dist/docs/example-agents/security-reviewer.html +4 -2
- package/dist/docs/example-agents/security-reviewer.md +265 -0
- package/dist/docs/example-agents/slack-agent.html +4 -2
- package/dist/docs/example-agents/slack-agent.md +142 -0
- package/dist/docs/example-agents/weather-agent.html +4 -2
- package/dist/docs/example-agents/weather-agent.md +296 -0
- package/dist/docs/guides/agent-to-agent.html +4 -2
- package/dist/docs/guides/agent-to-agent.md +113 -0
- package/dist/docs/guides/cloud-runtime.html +4 -2
- package/dist/docs/guides/cloud-runtime.md +131 -0
- package/dist/docs/guides/convert-automation.html +4 -2
- package/dist/docs/guides/convert-automation.md +171 -0
- package/dist/docs/guides/github.html +4 -2
- package/dist/docs/guides/github.md +287 -0
- package/dist/docs/guides/human-in-the-loop.html +4 -2
- package/dist/docs/guides/human-in-the-loop.md +126 -0
- package/dist/docs/guides/mcp-oauth.html +4 -2
- package/dist/docs/guides/mcp-oauth.md +159 -0
- package/dist/docs/guides/opentelemetry.html +4 -2
- package/dist/docs/guides/opentelemetry.md +209 -0
- package/dist/docs/guides/slack.html +4 -2
- package/dist/docs/guides/slack.md +337 -0
- package/dist/docs/guides/webhooks.html +4 -2
- package/dist/docs/guides/webhooks.md +463 -0
- package/dist/docs/hillclimbing.html +4 -2
- package/dist/docs/hillclimbing.md +88 -0
- package/dist/docs/index.html +4 -2
- package/dist/docs/index.md +179 -0
- package/dist/docs/llms-full.txt +11518 -0
- package/dist/docs/llms.txt +75 -0
- package/dist/docs/quickstart.html +4 -2
- package/dist/docs/quickstart.md +364 -0
- package/dist/docs/reference/agent-config.html +4 -2
- package/dist/docs/reference/agent-config.md +256 -0
- package/dist/docs/reference/artifacts.html +4 -2
- package/dist/docs/reference/artifacts.md +112 -0
- package/dist/docs/reference/channels.html +4 -2
- package/dist/docs/reference/channels.md +255 -0
- package/dist/docs/reference/cli.html +4 -2
- package/dist/docs/reference/cli.md +957 -0
- package/dist/docs/reference/connections.html +4 -2
- package/dist/docs/reference/connections.md +268 -0
- package/dist/docs/reference/hooks.html +4 -2
- package/dist/docs/reference/hooks.md +102 -0
- package/dist/docs/reference/http-api.html +4 -2
- package/dist/docs/reference/http-api.md +267 -0
- package/dist/docs/reference/instructions.html +4 -2
- package/dist/docs/reference/instructions.md +74 -0
- package/dist/docs/reference/playground.html +4 -2
- package/dist/docs/reference/playground.md +62 -0
- package/dist/docs/reference/project-layout.html +4 -2
- package/dist/docs/reference/project-layout.md +107 -0
- package/dist/docs/reference/prompt.html +4 -2
- package/dist/docs/reference/prompt.md +42 -0
- package/dist/docs/reference/schedules.html +4 -2
- package/dist/docs/reference/schedules.md +215 -0
- package/dist/docs/reference/sessions.html +4 -2
- package/dist/docs/reference/sessions.md +170 -0
- package/dist/docs/reference/skills.html +4 -2
- package/dist/docs/reference/skills.md +83 -0
- package/dist/docs/reference/subagents.html +4 -2
- package/dist/docs/reference/subagents.md +71 -0
- package/dist/docs/reference/tools.html +4 -2
- package/dist/docs/reference/tools.md +298 -0
- package/dist/docs/scaffolding-agents.html +4 -2
- package/dist/docs/scaffolding-agents.md +130 -0
- package/dist/docs/storage.html +4 -2
- package/dist/docs/storage.md +219 -0
- package/dist/docs/templates/agentic-owners.html +4 -2
- package/dist/docs/templates/agentic-owners.md +92 -0
- package/dist/docs/templates/demo.html +4 -2
- package/dist/docs/templates/demo.md +79 -0
- package/dist/docs/templates/pr-autofixer.html +4 -2
- package/dist/docs/templates/pr-autofixer.md +131 -0
- package/dist/docs/templates/security-reviewer.html +4 -2
- package/dist/docs/templates/security-reviewer.md +84 -0
- package/dist/docs/templates/triage.html +4 -2
- package/dist/docs/templates/triage.md +98 -0
- package/dist/docs/troubleshooting.html +4 -2
- package/dist/docs/troubleshooting.md +111 -0
- package/dist/internal/init-scaffold.d.ts.map +1 -1
- package/dist/internal/init-scaffold.js +1 -0
- package/dist/internal/playground/static.d.ts.map +1 -1
- package/dist/internal/playground/static.js +2 -0
- package/dist/internal/session-engine.d.ts.map +1 -1
- package/dist/internal/session-engine.js +3 -1
- package/package.json +2 -2
- package/skills/ab/SKILL.md +3 -0
- package/skills/create-agent/SKILL.md +3 -0
- package/skills/debug/SKILL.md +3 -0
- package/skills/evals/SKILL.md +3 -0
- package/skills/framework-map/SKILL.md +3 -0
- package/skills/github/SKILL.md +3 -0
- package/skills/hillclimb/SKILL.md +3 -0
- package/skills/mcp-auth/SKILL.md +3 -0
- package/skills/otel/SKILL.md +3 -0
- package/skills/setup-slack/SKILL.md +3 -0
- package/src/internal/init-scaffold.ts +1 -0
- package/src/internal/playground/static.ts +2 -0
- package/src/internal/session-engine.ts +4 -6
package/dist/docs/ab.md
ADDED
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
# Live A/B metrics
|
|
2
|
+
|
|
3
|
+
Use `defineAB` to compare variants on live agent sessions. New sessions
|
|
4
|
+
receive a sticky assignment in each enrolled experiment. The Agent SDK folds
|
|
5
|
+
their durable event streams into tool, token, failure, and wall-time
|
|
6
|
+
metrics. You can send cumulative samples to your metrics backend and
|
|
7
|
+
inspect aggregates in the playground.
|
|
8
|
+
|
|
9
|
+
`defineAB` compares live variants through sticky assignment,
|
|
10
|
+
instruction overlays, optional tool branches, and cumulative metrics.
|
|
11
|
+
Metric callbacks observe the result without approving, rejecting, or
|
|
12
|
+
failing a turn. Use [evals](/docs/evals.md) for pass/fail regression checks
|
|
13
|
+
on fixed inputs.
|
|
14
|
+
|
|
15
|
+
> [!NOTE]
|
|
16
|
+
> Import paths here use `@cursor/july/ab`. On projects still
|
|
17
|
+
> using `@anysphere/agent-serve`, swap the import. See
|
|
18
|
+
> [Run the CLI](/docs/index.md#run-the-cli) for the full rename table.
|
|
19
|
+
|
|
20
|
+
## Choose live A/B metrics or evals
|
|
21
|
+
|
|
22
|
+
Both features read the session event stream, but they answer different
|
|
23
|
+
questions.
|
|
24
|
+
|
|
25
|
+
| | Live A/B metrics | Evals |
|
|
26
|
+
| --- | --- | --- |
|
|
27
|
+
| Question | How do variants compare on live sessions? | Does the agent still meet a fixed contract? |
|
|
28
|
+
| Location | `agent/ab.ts` or `agent/ab/<name>.ts` | `evals/**/*.eval.ts` |
|
|
29
|
+
| Input | Dev or production traffic | Frozen prompts and fixtures |
|
|
30
|
+
| Output | Cumulative metrics by session and arm | Pass/fail assertions |
|
|
31
|
+
| How it runs | Automatically on new live sessions | `agent-sdk eval` |
|
|
32
|
+
|
|
33
|
+
There is no `agent-sdk ab` command or assertion API.
|
|
34
|
+
|
|
35
|
+
## Define an experiment
|
|
36
|
+
|
|
37
|
+
Author one experiment in `agent/ab.ts`, add more under
|
|
38
|
+
`agent/ab/<name>.ts`, or use both forms. Each file defines one
|
|
39
|
+
experiment. The experiment name comes from `name` when set. Otherwise,
|
|
40
|
+
The Agent SDK uses `ab` for `agent/ab.ts` and the file stem for files under
|
|
41
|
+
`agent/ab/`.
|
|
42
|
+
|
|
43
|
+
```ts
|
|
44
|
+
// agent/ab/concise-weather.ts
|
|
45
|
+
import {
|
|
46
|
+
defineAB,
|
|
47
|
+
splitBySessionHash,
|
|
48
|
+
} from "@cursor/july/ab";
|
|
49
|
+
|
|
50
|
+
export default defineAB({
|
|
51
|
+
name: "concise-weather",
|
|
52
|
+
variants: {
|
|
53
|
+
control: {
|
|
54
|
+
label: "Baseline",
|
|
55
|
+
},
|
|
56
|
+
treatment: {
|
|
57
|
+
label: "Short replies",
|
|
58
|
+
description: "Adds a one-paragraph response limit.",
|
|
59
|
+
instructions: "Keep weather replies to one short paragraph.",
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
split: splitBySessionHash({
|
|
63
|
+
weights: { control: 1, treatment: 1 },
|
|
64
|
+
holdout: 0.1,
|
|
65
|
+
}),
|
|
66
|
+
derive: {
|
|
67
|
+
weatherCalls: (event) =>
|
|
68
|
+
event.type === "action.result" &&
|
|
69
|
+
event.data.toolName === "get_weather"
|
|
70
|
+
? 1
|
|
71
|
+
: null,
|
|
72
|
+
},
|
|
73
|
+
onSample(sample) {
|
|
74
|
+
console.log(
|
|
75
|
+
sample.experiment,
|
|
76
|
+
sample.variant,
|
|
77
|
+
sample.metrics.toolCalls,
|
|
78
|
+
sample.metrics.wallTimeMs
|
|
79
|
+
);
|
|
80
|
+
},
|
|
81
|
+
});
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Every definition needs:
|
|
85
|
+
|
|
86
|
+
- At least two variants. Variant keys cannot be empty or contain `/` or
|
|
87
|
+
`\`.
|
|
88
|
+
- A `split` function that returns a variant key or `null`.
|
|
89
|
+
- An `onSample` callback for completed or failed turns.
|
|
90
|
+
|
|
91
|
+
`label` and `description` appear with the arm in result surfaces.
|
|
92
|
+
`instructions` changes the prompt for sessions in that arm. `derive`
|
|
93
|
+
adds custom counters.
|
|
94
|
+
|
|
95
|
+
Duplicate experiment names are validation errors. Check discovery
|
|
96
|
+
before you serve:
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
agent-sdk validate --dir .
|
|
100
|
+
agent-sdk info --dir . --json
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The `abs` field in `info` lists the discovered experiment names.
|
|
104
|
+
|
|
105
|
+
## Assign sticky variants
|
|
106
|
+
|
|
107
|
+
Enrollment happens once, when a live session is created and before its
|
|
108
|
+
first turn:
|
|
109
|
+
|
|
110
|
+
1. The Agent SDK records `session.started`.
|
|
111
|
+
2. Each experiment runs its `split` function.
|
|
112
|
+
3. The Agent SDK records one durable `ab.assigned` event per experiment.
|
|
113
|
+
4. The selected arms become available on `session.abs`.
|
|
114
|
+
5. Variant instruction overlays reach the first model turn.
|
|
115
|
+
|
|
116
|
+
A split can return a variant key or `null`. A null assignment is a
|
|
117
|
+
sticky skip for that experiment. It increments the experiment's
|
|
118
|
+
`skipped` total, still appears in the snapshot's `sessions` list with
|
|
119
|
+
`variant: null`, and does not collect arm metrics or call `onSample`.
|
|
120
|
+
|
|
121
|
+
Use the split helper that matches your rollout:
|
|
122
|
+
|
|
123
|
+
| Helper | Behavior |
|
|
124
|
+
| --- | --- |
|
|
125
|
+
| `splitBySessionHash({ weights?, holdout?, salt? })` | Hashes the session id into a reproducible arm; the recommended default |
|
|
126
|
+
| `splitByRandom({ weights?, holdout? })` | Draws once when the session starts, then persists the result |
|
|
127
|
+
| `splitAlways("control")` | Pins every new session to one arm |
|
|
128
|
+
| `splitNone()` | Skips every new session without deleting the experiment |
|
|
129
|
+
| `splitIf(predicate, inner)` | Runs `inner` only when the predicate passes |
|
|
130
|
+
| Custom `split(ctx)` | Returns a declared variant key or `null` |
|
|
131
|
+
|
|
132
|
+
The split context includes the agent name, channel id, session info,
|
|
133
|
+
experiment name, and declared variant keys. For example, enroll only
|
|
134
|
+
Slack sessions:
|
|
135
|
+
|
|
136
|
+
```ts
|
|
137
|
+
split: splitIf(
|
|
138
|
+
(ctx) => ctx.channel.id === "slack",
|
|
139
|
+
splitBySessionHash()
|
|
140
|
+
),
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Weights default to equal. Non-positive weights leave an arm out of the
|
|
144
|
+
draw, and at least one arm must have a positive weight. `holdout` is the
|
|
145
|
+
fraction of sessions assigned `null`, from `0` through `1`. Change
|
|
146
|
+
`salt` to reshuffle future hash assignments without renaming the
|
|
147
|
+
experiment.
|
|
148
|
+
|
|
149
|
+
If a custom split throws or returns an unknown variant, the Agent SDK logs
|
|
150
|
+
the error and records `variant: null`. The failed decision becomes a
|
|
151
|
+
sticky skip instead of breaking the session.
|
|
152
|
+
|
|
153
|
+
Enrollment only applies to new sessions. Adding an experiment does not
|
|
154
|
+
assign existing conversations. Follow-ups keep the session's original
|
|
155
|
+
arms. Keep experiment names and variant keys stable while you collect
|
|
156
|
+
and compare results.
|
|
157
|
+
|
|
158
|
+
## Change behavior by variant
|
|
159
|
+
|
|
160
|
+
Variant instructions are appended to the agent's base instructions.
|
|
161
|
+
Local sessions receive the merged instructions in `AGENTS.md` before
|
|
162
|
+
every turn. Cloud sessions receive them in the first-turn preamble
|
|
163
|
+
only. For cloud follow-ups, branch through `session.abs` when the arm
|
|
164
|
+
must remain visible to deterministic behavior.
|
|
165
|
+
|
|
166
|
+
Tools can branch on the assignment through `ctx.session.abs`. Hooks can
|
|
167
|
+
read the same field for logging or export:
|
|
168
|
+
|
|
169
|
+
```ts
|
|
170
|
+
const treatment =
|
|
171
|
+
ctx.session.abs?.["concise-weather"] === "treatment";
|
|
172
|
+
|
|
173
|
+
if (treatment) {
|
|
174
|
+
return conciseWeatherResult;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
return baselineWeatherResult;
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
This makes the assignment available to deterministic code as well as
|
|
181
|
+
the model prompt. Use both patterns together when one experiment must
|
|
182
|
+
steer the prompt and host code at once.
|
|
183
|
+
|
|
184
|
+
`defineAB` does not select a different model or runtime for each arm.
|
|
185
|
+
Keep those settings in `agent/agent.ts`, or write explicit host logic
|
|
186
|
+
when your experiment needs another behavior lever.
|
|
187
|
+
|
|
188
|
+
The split and selected arm can affect agent behavior. `derive` and
|
|
189
|
+
`onSample` only observe the resulting event stream. Errors in either
|
|
190
|
+
callback are logged and never fail the turn.
|
|
191
|
+
|
|
192
|
+
## Collect built-in and custom metrics
|
|
193
|
+
|
|
194
|
+
Metrics accumulate for each session and experiment. When one session
|
|
195
|
+
joins several experiments, every enrolled experiment folds the same
|
|
196
|
+
turn and tool events into its own counters.
|
|
197
|
+
|
|
198
|
+
| Metric | How the Agent SDK calculates it |
|
|
199
|
+
| --- | --- |
|
|
200
|
+
| `turns` | Adds one on `turn.completed` or `turn.failed` |
|
|
201
|
+
| `turnFailures` | Adds one on `turn.failed` |
|
|
202
|
+
| `toolCalls` | Adds one for each `action.result` |
|
|
203
|
+
| `toolErrors` | Adds one when `action.result.data.isError` is true |
|
|
204
|
+
| `inputTokens`, `outputTokens` | Adds usage from completed turns |
|
|
205
|
+
| `cacheReadTokens`, `cacheWriteTokens` | Adds cache usage from completed turns |
|
|
206
|
+
| `costUsd` | Sums the estimated turn cost recorded on `turn.completed` (turns whose model has no known rates contribute 0) |
|
|
207
|
+
| `wallTimeMs` | Sums the time from `turn.started` to its completed or failed event |
|
|
208
|
+
| `custom` | Sums finite numeric deltas returned by `derive` |
|
|
209
|
+
|
|
210
|
+
`onSample` fires after every `turn.completed` and `turn.failed` event
|
|
211
|
+
for an enrolled arm. The sample contains:
|
|
212
|
+
|
|
213
|
+
| Field | Value |
|
|
214
|
+
| --- | --- |
|
|
215
|
+
| `experiment` | Experiment name |
|
|
216
|
+
| `variant`, `variantLabel?` | Sticky arm and optional display label |
|
|
217
|
+
| `sessionId`, `channelId` | Source session |
|
|
218
|
+
| `metrics` | Cumulative metrics through this turn |
|
|
219
|
+
| `reason` | `turn.completed` or `turn.failed` |
|
|
220
|
+
| `at` | Terminal event timestamp |
|
|
221
|
+
|
|
222
|
+
The metrics are cumulative, not per-turn deltas. A second sample from
|
|
223
|
+
the same session includes the first turn's counts.
|
|
224
|
+
|
|
225
|
+
Each `derive` extractor runs on every session event for its enrolled
|
|
226
|
+
experiment, including streamed `message.appended` events. Keep it
|
|
227
|
+
synchronous and cheap. Return a finite number to add a delta, or
|
|
228
|
+
`null` to skip the event. Send samples to your metrics service from
|
|
229
|
+
`onSample`; do not perform network or disk work in `derive`.
|
|
230
|
+
|
|
231
|
+
Skipped sessions never call `onSample`. Errors from `derive` or
|
|
232
|
+
`onSample` are logged, then metric collection continues.
|
|
233
|
+
|
|
234
|
+
## Inspect assignments and results
|
|
235
|
+
|
|
236
|
+
Open the playground's **A/Bs** tab to see aggregate arm totals and
|
|
237
|
+
per-session assignments. The tab reads `GET /v1/abs`.
|
|
238
|
+
|
|
239
|
+
The response has two views of the same durable data:
|
|
240
|
+
|
|
241
|
+
| Field | Contents |
|
|
242
|
+
| --- | --- |
|
|
243
|
+
| `experiments` | Declared variants, skipped-session count, arm session counts, and aggregate metrics |
|
|
244
|
+
| `sessions` | Visible sessions with their assignments and cumulative metrics |
|
|
245
|
+
|
|
246
|
+
`GET /v1/abs` returns sessions visible to the current principal by
|
|
247
|
+
default. In `--dev`, loopback requests include every session. Add
|
|
248
|
+
`--allow-anonymous` to include every session from non-loopback callers
|
|
249
|
+
too. This include-all behavior can still apply to `GET /v1/abs` in dev
|
|
250
|
+
when bearer or custom auth keeps `GET /v1/sessions` owner-scoped.
|
|
251
|
+
|
|
252
|
+
Session `events.ndjson` is the source of truth for assignment + fold.
|
|
253
|
+
`GET /v1/abs` recomputes aggregates from those logs. Any
|
|
254
|
+
`agent/storage.ts` exports samples and snapshots durably: an authored
|
|
255
|
+
`abs` table when the backend has a native shape for it, or the table
|
|
256
|
+
derived over the KV core otherwise. See
|
|
257
|
+
[Storage](/docs/storage.md#eval-and-a-b-tables).
|
|
258
|
+
|
|
259
|
+
## Configure the playground fold window
|
|
260
|
+
|
|
261
|
+
Assignments and foldable metrics already persist in each session's
|
|
262
|
+
`events.ndjson` under `--state-root`. The optional `agent/ab.config.ts`
|
|
263
|
+
only caps how many sessions the playground and `GET /v1/abs` fold:
|
|
264
|
+
|
|
265
|
+
```ts
|
|
266
|
+
import { defineABConfig } from "@cursor/july/ab";
|
|
267
|
+
|
|
268
|
+
export default defineABConfig({
|
|
269
|
+
// Optional — defaults to 200. Only affects GET /v1/abs / A/Bs tab.
|
|
270
|
+
maxPlaygroundSessions: 500,
|
|
271
|
+
});
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
`maxPlaygroundSessions` keeps the newest sessions in the fold. It does
|
|
275
|
+
not prune session logs or change assignment. For export to S3, a DB, or
|
|
276
|
+
your metrics vendor, send samples from `onSample` or declare a storage
|
|
277
|
+
`abs` table.
|
|
278
|
+
|
|
279
|
+
## Keep assignments durable
|
|
280
|
+
|
|
281
|
+
The append-only `events.ndjson` stream is the source of truth. Each
|
|
282
|
+
`ab.assigned` event persists a variant key or null skip. Built-in
|
|
283
|
+
metrics come from the turn and tool events that follow it.
|
|
284
|
+
|
|
285
|
+
After a server restart or a parked session resumes, the live collector
|
|
286
|
+
replays the stream to rebuild cumulative counters. Replay does not call
|
|
287
|
+
`onSample` (or write to the storage `abs` table) for historical turns.
|
|
288
|
+
Only a new completed or failed turn emits another sample.
|
|
289
|
+
|
|
290
|
+
The snapshot API also replays `derive` across the full stream, so
|
|
291
|
+
custom totals match the current extractor. Changing a derive function
|
|
292
|
+
can change historical snapshot totals. Treat metric definitions as
|
|
293
|
+
versioned experiment code.
|
|
294
|
+
|
|
295
|
+
## Keep eval traffic separate
|
|
296
|
+
|
|
297
|
+
Sessions created by `agent-sdk eval` and the playground Evals runner use
|
|
298
|
+
`purpose: "eval"`. They skip A/B enrollment entirely:
|
|
299
|
+
|
|
300
|
+
- No split function runs.
|
|
301
|
+
- No `ab.assigned` event is recorded.
|
|
302
|
+
- No `onSample` callback fires.
|
|
303
|
+
- The session is omitted from `GET /v1/abs`.
|
|
304
|
+
|
|
305
|
+
Ordinary chat, `agent-sdk run`, Slack, GitHub, and other channel sessions
|
|
306
|
+
use the live purpose. You do not need `splitIf` to exclude eval traffic.
|
|
307
|
+
|
|
308
|
+
## Know the boundaries
|
|
309
|
+
|
|
310
|
+
`defineAB` provides sticky assignment, variant instructions,
|
|
311
|
+
`session.abs` for tools, cumulative metrics, and local inspection. It
|
|
312
|
+
does not provide:
|
|
313
|
+
|
|
314
|
+
- A test command, assertion API, or pass/fail result
|
|
315
|
+
- Statistical significance calculations
|
|
316
|
+
- An experiment rollout or lifecycle service
|
|
317
|
+
- Per-variant model or runtime configuration
|
|
318
|
+
- A built-in analytics warehouse (bring your own via `onSample` or the
|
|
319
|
+
storage `abs` table)
|
|
320
|
+
|
|
321
|
+
Use [evals](/docs/evals.md) to protect known behavior. Use `onSample` or a
|
|
322
|
+
storage `abs` table when you need sample/snapshot exports beyond the
|
|
323
|
+
session event log.
|
|
324
|
+
|
|
325
|
+
## What's next
|
|
326
|
+
|
|
327
|
+
Continue with these pages:
|
|
328
|
+
|
|
329
|
+
- [Evals](/docs/evals.md): pass/fail regression checks on fixed inputs
|
|
330
|
+
- [Hillclimbing](/docs/hillclimbing.md): improve an agent against fixed
|
|
331
|
+
fixtures
|
|
332
|
+
- [Hooks](/docs/reference/hooks.md): other event-stream consumers
|
|
333
|
+
- [Sessions and streaming](/docs/reference/sessions.md): the
|
|
334
|
+
`ab.assigned` event and durable log
|
|
335
|
+
- [Playground](/docs/reference/playground.md): the A/Bs tab
|
|
336
|
+
- [HTTP API](/docs/reference/http-api.md): `GET /v1/abs`
|
|
337
|
+
- [Live A/B metrics skill](https://github.com/cursor/cursor/blob/main/packages/agent-serve/skills/ab/SKILL.md): have a coding agent
|
|
338
|
+
wire an experiment
|
|
@@ -16,9 +16,11 @@
|
|
|
16
16
|
<link rel="modulepreload" href="/docs/assets/building-with-agents.md.DH8A_cHA.lean.js">
|
|
17
17
|
<script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
|
|
18
18
|
<script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
|
|
19
|
+
<link rel="alternate" type="text/plain" href="/docs/llms.txt">
|
|
20
|
+
<link rel="alternate" type="text/markdown" href="/docs/building-with-agents.md">
|
|
19
21
|
</head>
|
|
20
22
|
<body>
|
|
21
|
-
<div id="app"><div class="Layout" data-v-282c430e><!--[--><!--]--><!--[--><span tabindex="-1" data-v-af8643bc></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-af8643bc>Skip to content</a><!--]--><!----><header class="VPNav" data-v-282c430e data-v-4751689a><div class="VPNavBar" data-v-4751689a data-v-d2a336f3><div class="wrapper" data-v-d2a336f3><div class="container" data-v-d2a336f3><div class="title" data-v-d2a336f3><div class="VPNavBarTitle has-sidebar" data-v-d2a336f3 data-v-3a787a7b><a class="title" href="/docs/" data-v-3a787a7b><!--[--><!--]--><!----><span data-v-3a787a7b>Agent SDK</span><!--[--><!--[--><!--[--><!--[--><span class="agent-sdk-version" title="@cursor/july 0.1.91" data-v-c26a5f4f>0.1.91</span><!--]--><!--]--><!--]--><!--]--></a></div></div><div class="content" data-v-d2a336f3><div class="content-body" data-v-d2a336f3><!--[--><!--]--><div class="VPNavBarSearch search" data-v-d2a336f3><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-d2a336f3 data-v-b60c0a58><span id="main-nav-aria-label" class="visually-hidden" data-v-b60c0a58> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/quickstart.html" tabindex="0" data-v-b60c0a58 data-v-74de87b9><!--[--><span data-v-74de87b9>Quickstart</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/guides/webhooks.html" tabindex="0" data-v-b60c0a58 data-v-74de87b9><!--[--><span data-v-74de87b9>Guides</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/example-agents/" tabindex="0" data-v-b60c0a58 data-v-74de87b9><!--[--><span data-v-74de87b9>Examples</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/reference/project-layout.html" tabindex="0" data-v-b60c0a58 data-v-74de87b9><!--[--><span data-v-74de87b9>Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-d2a336f3 data-v-d559fc20><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-d559fc20 data-v-deaafa36 data-v-88da49d2><span class="check" data-v-88da49d2><span class="icon" data-v-88da49d2><!--[--><span class="vpi-sun sun" data-v-deaafa36></span><span class="vpi-moon moon" data-v-deaafa36></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-d2a336f3 data-v-75b97eb1 data-v-ed271fd4><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-ed271fd4><span class="vpi-more-horizontal icon" data-v-ed271fd4></span></button><div class="menu" data-v-ed271fd4><div class="VPMenu" data-v-ed271fd4 data-v-505057d2><!----><!--[--><!--[--><!----><div class="group" data-v-75b97eb1><div class="item appearance" data-v-75b97eb1><p class="label" data-v-75b97eb1>Appearance</p><div class="appearance-action" data-v-75b97eb1><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-75b97eb1 data-v-deaafa36 data-v-88da49d2><span class="check" data-v-88da49d2><span class="icon" data-v-88da49d2><!--[--><span class="vpi-sun sun" data-v-deaafa36></span><span class="vpi-moon moon" data-v-deaafa36></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-d2a336f3 data-v-966f1ac7><span class="container" data-v-966f1ac7><span class="top" data-v-966f1ac7></span><span class="middle" data-v-966f1ac7></span><span class="bottom" data-v-966f1ac7></span></span></button></div></div></div></div><div class="divider" data-v-d2a336f3><div class="divider-line" data-v-d2a336f3></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-282c430e data-v-f96f8409><div class="container" data-v-f96f8409><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-f96f8409><span class="vpi-align-left menu-icon" data-v-f96f8409></span><span class="menu-text" data-v-f96f8409>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-f96f8409 data-v-1bc67f4b><button data-v-1bc67f4b>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-282c430e data-v-3d1258d2><div class="curtain" data-v-3d1258d2></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-3d1258d2><span class="visually-hidden" id="sidebar-aria-label" data-v-3d1258d2> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Getting started</h2><!----></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/quickstart.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Quickstart</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/scaffolding-agents.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Scaffold an agent with Cursor</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/convert-automation.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Convert a Cursor Automation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/concepts.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Concepts</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Templates</h2><!----></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/templates/demo.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Demo agent</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/templates/security-reviewer.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Security reviewer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/templates/agentic-owners.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Agentic Owners</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/templates/pr-autofixer.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>PR autofixer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/templates/triage.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Triage agent</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0 has-active" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Improving agents</h2><!----></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/building-with-agents.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Building agents with agents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/evals.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Evals</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/ab.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Live A/B metrics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/storage.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Storage</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/hillclimbing.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Hillclimbing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Guides</h2><!----></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/webhooks.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Webhooks & custom channels</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/github.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>GitHub</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/slack.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Slack</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/human-in-the-loop.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Human-in-the-loop approvals</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/agent-to-agent.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Agent-to-agent</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/mcp-oauth.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Host MCP OAuth</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/cloud-runtime.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Cloud runtime</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/opentelemetry.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>OpenTelemetry</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Example agents</h2><!----></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Choose an example</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/weather-agent.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Weather agent</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/slack-agent.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Slack agent</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/concierge.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Concierge</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/benny.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Playbook router</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/oncall.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Alert investigator</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/bugbot.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>PR evidence reviewer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/approval-buddy.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Approval Buddy</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/security-reviewer.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Security Reviewer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/fsd.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Remote PR coordinator</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/knowledge-base.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Knowledge base</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/codebase-wiki.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Codebase wiki</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/codeowners-review.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Codeowners review</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Operating</h2><!----></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/deployment.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Deployment</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/troubleshooting.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Fix common problems</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0 collapsible" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Reference</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-61fcdc7c><span class="vpi-chevron-right caret-icon" data-v-61fcdc7c></span></div></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/project-layout.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Project layout</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/agent-config.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Agent config</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/instructions.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Instructions</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/tools.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Tools</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/prompt.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>prompt</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/skills.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Skills</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/connections.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>MCP Connections</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/subagents.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Subagents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/channels.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Channels</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/schedules.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Schedules & reminders</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/hooks.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Hooks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/artifacts.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Artifacts</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/sessions.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Sessions & streaming</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/playground.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Playground</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/cli.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>CLI</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/http-api.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>HTTP API</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-282c430e data-v-3b8d81ee><div class="VPDoc has-sidebar has-aside" data-v-3b8d81ee data-v-2e506a34><!--[--><!--]--><div class="container" data-v-2e506a34><div class="aside" data-v-2e506a34><div class="aside-curtain" data-v-2e506a34></div><div class="aside-container" data-v-2e506a34><div class="aside-content" data-v-2e506a34><div class="VPDocAside" data-v-2e506a34 data-v-b485a79e><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-b485a79e data-v-cc6b345c><div class="content" data-v-cc6b345c><div class="outline-marker" data-v-cc6b345c></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-cc6b345c>On this page</div><ul class="VPDocOutlineItem root" data-v-cc6b345c data-v-4f0215bf><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-b485a79e></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-2e506a34><div class="content-container" data-v-2e506a34><!--[--><!--]--><main class="main" data-v-2e506a34><div style="position:relative;" class="vp-doc _docs_building-with-agents" data-v-2e506a34><div><h1 id="building-agents-with-agents" tabindex="-1">Building agents with agents <a class="header-anchor" href="#building-agents-with-agents" aria-label="Permalink to "Building agents with agents""></a></h1><p>Give a coding agent the goal. The built-in skills guide it through scaffolding, channels, verification, evals, and measured improvement.</p><h2 id="what-can-a-coding-agent-build-for-me" tabindex="-1">What can a coding agent build for me? <a class="header-anchor" href="#what-can-a-coding-agent-build-for-me" aria-label="Permalink to "What can a coding agent build for me?""></a></h2><p>A coding agent can take an Agent SDK project from an idea to a verified first version. It can:</p><ul><li>Create the project and choose the smallest useful surface</li><li>Add tools, skills, channels, MCP connections, and approval gates</li><li>Run a real turn and inspect its trace</li><li>Add evals for the behavior you want to keep</li><li>Improve the agent against fixed inputs</li></ul><p>This works because the project surface lives in files. The CLI also returns structured JSON and records each run as an NDJSON trace. Your coding agent can inspect what the framework discovered, run the agent, and verify the result without reading terminal prose.</p><h2 id="how-do-i-create-an-agent-with-the-built-in-skill" tabindex="-1">How do I create an agent with the built-in skill? <a class="header-anchor" href="#how-do-i-create-an-agent-with-the-built-in-skill" aria-label="Permalink to "How do I create an agent with the built-in skill?""></a></h2><p>Have the coding agent read <a href="./../skills/create-agent/SKILL.html"><code>skills/create-agent/SKILL.md</code></a> (in the monorepo: <code>packages/agent-serve/skills/create-agent/SKILL.md</code>) and follow it.</p><p>The skill asks about your agent's purpose, runtime, model, channels, MCP connections, and capabilities. It then shows you a plan, writes the project, and verifies the result.</p><p>For example:</p><blockquote><p>Use the Agent SDK create-agent skill to build a PR triage agent reachable through GitHub. It should summarize failed checks, require approval before posting a review, and include one smoke eval.</p></blockquote><p>The skill starts with a small surface: short instructions, one or two tools, the channels you chose, and a smoke eval. You can add more after the first end-to-end turn works.</p><h2 id="which-built-in-skill-should-i-use" tabindex="-1">Which built-in skill should I use? <a class="header-anchor" href="#which-built-in-skill-should-i-use" aria-label="Permalink to "Which built-in skill should I use?""></a></h2><p>The package ships task-specific guides under <a href="./../skills/"><code>skills/</code></a>:</p><table tabindex="0"><thead><tr><th>What you want to do</th><th>Skill</th></tr></thead><tbody><tr><td>Understand the project layout and runtimes</td><td><a href="./../skills/framework-map/SKILL.html"><code>framework-map</code></a></td></tr><tr><td>Create and verify a new agent</td><td><a href="./../skills/create-agent/SKILL.html"><code>create-agent</code></a></td></tr><tr><td>Write fixtures and regression checks</td><td><a href="./../skills/evals/SKILL.html"><code>evals</code></a></td></tr><tr><td>Live A/B metrics on traffic (<code>defineAB</code>)</td><td><a href="./../skills/ab/SKILL.html"><code>ab</code></a></td></tr><tr><td>Improve an agent against fixed inputs</td><td><a href="./../skills/hillclimb/SKILL.html"><code>hillclimb</code></a></td></tr><tr><td>Add GitHub webhooks and replay events</td><td><a href="./../skills/github/SKILL.html"><code>github</code></a></td></tr><tr><td>Connect an agent to Slack</td><td><a href="./../skills/setup-slack/SKILL.html"><code>setup-slack</code></a></td></tr><tr><td>Diagnose a local run</td><td><a href="./../skills/debug/SKILL.html"><code>debug</code></a></td></tr></tbody></table><p>Point your coding agent at the matching <code>SKILL.md</code>. The guide contains the workflow, commands, and common mistakes for that task.</p><div class="note custom-block github-alert"><p class="custom-block-title">NOTE</p><p>The skill bodies use the current <code>agent-serve</code> CLI names. This guide uses the upcoming <code>agent-sdk</code> names. See <a href="/docs/#run-the-cli">Run the CLI</a> for the full rename table.</p></div><h2 id="how-does-a-coding-agent-verify-its-work" tabindex="-1">How does a coding agent verify its work? <a class="header-anchor" href="#how-does-a-coding-agent-verify-its-work" aria-label="Permalink to "How does a coding agent verify its work?""></a></h2><p>The coding agent should discover the project, test each server tool, run a real turn, and finish with evals:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span></span>
|
|
23
|
+
<div id="app"><div class="Layout" data-v-282c430e><!--[--><!--]--><!--[--><span tabindex="-1" data-v-af8643bc></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-af8643bc>Skip to content</a><!--]--><!----><header class="VPNav" data-v-282c430e data-v-4751689a><div class="VPNavBar" data-v-4751689a data-v-d2a336f3><div class="wrapper" data-v-d2a336f3><div class="container" data-v-d2a336f3><div class="title" data-v-d2a336f3><div class="VPNavBarTitle has-sidebar" data-v-d2a336f3 data-v-3a787a7b><a class="title" href="/docs/" data-v-3a787a7b><!--[--><!--]--><!----><span data-v-3a787a7b>Agent SDK</span><!--[--><!--[--><!--[--><!--[--><span class="agent-sdk-version" title="@cursor/july 0.1.92" data-v-c26a5f4f>0.1.92</span><!--]--><!--]--><!--]--><!--]--></a></div></div><div class="content" data-v-d2a336f3><div class="content-body" data-v-d2a336f3><!--[--><!--]--><div class="VPNavBarSearch search" data-v-d2a336f3><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-d2a336f3 data-v-b60c0a58><span id="main-nav-aria-label" class="visually-hidden" data-v-b60c0a58> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/quickstart.html" tabindex="0" data-v-b60c0a58 data-v-74de87b9><!--[--><span data-v-74de87b9>Quickstart</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/guides/webhooks.html" tabindex="0" data-v-b60c0a58 data-v-74de87b9><!--[--><span data-v-74de87b9>Guides</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/example-agents/" tabindex="0" data-v-b60c0a58 data-v-74de87b9><!--[--><span data-v-74de87b9>Examples</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/reference/project-layout.html" tabindex="0" data-v-b60c0a58 data-v-74de87b9><!--[--><span data-v-74de87b9>Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-d2a336f3 data-v-d559fc20><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-d559fc20 data-v-deaafa36 data-v-88da49d2><span class="check" data-v-88da49d2><span class="icon" data-v-88da49d2><!--[--><span class="vpi-sun sun" data-v-deaafa36></span><span class="vpi-moon moon" data-v-deaafa36></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-d2a336f3 data-v-75b97eb1 data-v-ed271fd4><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-ed271fd4><span class="vpi-more-horizontal icon" data-v-ed271fd4></span></button><div class="menu" data-v-ed271fd4><div class="VPMenu" data-v-ed271fd4 data-v-505057d2><!----><!--[--><!--[--><!----><div class="group" data-v-75b97eb1><div class="item appearance" data-v-75b97eb1><p class="label" data-v-75b97eb1>Appearance</p><div class="appearance-action" data-v-75b97eb1><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-75b97eb1 data-v-deaafa36 data-v-88da49d2><span class="check" data-v-88da49d2><span class="icon" data-v-88da49d2><!--[--><span class="vpi-sun sun" data-v-deaafa36></span><span class="vpi-moon moon" data-v-deaafa36></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-d2a336f3 data-v-966f1ac7><span class="container" data-v-966f1ac7><span class="top" data-v-966f1ac7></span><span class="middle" data-v-966f1ac7></span><span class="bottom" data-v-966f1ac7></span></span></button></div></div></div></div><div class="divider" data-v-d2a336f3><div class="divider-line" data-v-d2a336f3></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-282c430e data-v-f96f8409><div class="container" data-v-f96f8409><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-f96f8409><span class="vpi-align-left menu-icon" data-v-f96f8409></span><span class="menu-text" data-v-f96f8409>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-f96f8409 data-v-1bc67f4b><button data-v-1bc67f4b>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-282c430e data-v-3d1258d2><div class="curtain" data-v-3d1258d2></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-3d1258d2><span class="visually-hidden" id="sidebar-aria-label" data-v-3d1258d2> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Getting started</h2><!----></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/quickstart.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Quickstart</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/scaffolding-agents.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Scaffold an agent with Cursor</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/convert-automation.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Convert a Cursor Automation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/concepts.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Concepts</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Templates</h2><!----></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/templates/demo.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Demo agent</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/templates/security-reviewer.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Security reviewer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/templates/agentic-owners.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Agentic Owners</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/templates/pr-autofixer.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>PR autofixer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/templates/triage.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Triage agent</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0 has-active" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Improving agents</h2><!----></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/building-with-agents.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Building agents with agents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/evals.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Evals</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/ab.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Live A/B metrics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/storage.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Storage</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/hillclimbing.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Hillclimbing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Guides</h2><!----></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/webhooks.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Webhooks & custom channels</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/github.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>GitHub</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/slack.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Slack</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/human-in-the-loop.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Human-in-the-loop approvals</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/agent-to-agent.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Agent-to-agent</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/mcp-oauth.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Host MCP OAuth</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/cloud-runtime.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Cloud runtime</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/guides/opentelemetry.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>OpenTelemetry</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Example agents</h2><!----></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Choose an example</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/weather-agent.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Weather agent</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/slack-agent.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Slack agent</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/concierge.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Concierge</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/benny.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Playbook router</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/oncall.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Alert investigator</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/bugbot.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>PR evidence reviewer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/approval-buddy.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Approval Buddy</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/security-reviewer.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Security Reviewer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/fsd.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Remote PR coordinator</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/knowledge-base.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Knowledge base</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/codebase-wiki.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Codebase wiki</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/example-agents/codeowners-review.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Codeowners review</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Operating</h2><!----></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/deployment.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Deployment</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/troubleshooting.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Fix common problems</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-f2306e18><section class="VPSidebarItem level-0 collapsible" data-v-f2306e18 data-v-61fcdc7c><div class="item" role="button" tabindex="0" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><h2 class="text" data-v-61fcdc7c>Reference</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-61fcdc7c><span class="vpi-chevron-right caret-icon" data-v-61fcdc7c></span></div></div><div class="items" data-v-61fcdc7c><!--[--><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/project-layout.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Project layout</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/agent-config.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Agent config</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/instructions.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Instructions</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/tools.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Tools</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/prompt.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>prompt</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/skills.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Skills</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/connections.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>MCP Connections</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/subagents.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Subagents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/channels.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Channels</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/schedules.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Schedules & reminders</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/hooks.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Hooks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/artifacts.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Artifacts</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/sessions.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Sessions & streaming</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/playground.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>Playground</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/cli.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>CLI</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-61fcdc7c><div class="item" data-v-61fcdc7c><div class="indicator" data-v-61fcdc7c></div><a class="VPLink link link" href="/docs/reference/http-api.html" data-v-61fcdc7c><!--[--><p class="text" data-v-61fcdc7c>HTTP API</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-282c430e data-v-3b8d81ee><div class="VPDoc has-sidebar has-aside" data-v-3b8d81ee data-v-2e506a34><!--[--><!--]--><div class="container" data-v-2e506a34><div class="aside" data-v-2e506a34><div class="aside-curtain" data-v-2e506a34></div><div class="aside-container" data-v-2e506a34><div class="aside-content" data-v-2e506a34><div class="VPDocAside" data-v-2e506a34 data-v-b485a79e><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-b485a79e data-v-cc6b345c><div class="content" data-v-cc6b345c><div class="outline-marker" data-v-cc6b345c></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-cc6b345c>On this page</div><ul class="VPDocOutlineItem root" data-v-cc6b345c data-v-4f0215bf><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-b485a79e></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-2e506a34><div class="content-container" data-v-2e506a34><!--[--><!--]--><main class="main" data-v-2e506a34><div style="position:relative;" class="vp-doc _docs_building-with-agents" data-v-2e506a34><div><h1 id="building-agents-with-agents" tabindex="-1">Building agents with agents <a class="header-anchor" href="#building-agents-with-agents" aria-label="Permalink to "Building agents with agents""></a></h1><p>Give a coding agent the goal. The built-in skills guide it through scaffolding, channels, verification, evals, and measured improvement.</p><h2 id="what-can-a-coding-agent-build-for-me" tabindex="-1">What can a coding agent build for me? <a class="header-anchor" href="#what-can-a-coding-agent-build-for-me" aria-label="Permalink to "What can a coding agent build for me?""></a></h2><p>A coding agent can take an Agent SDK project from an idea to a verified first version. It can:</p><ul><li>Create the project and choose the smallest useful surface</li><li>Add tools, skills, channels, MCP connections, and approval gates</li><li>Run a real turn and inspect its trace</li><li>Add evals for the behavior you want to keep</li><li>Improve the agent against fixed inputs</li></ul><p>This works because the project surface lives in files. The CLI also returns structured JSON and records each run as an NDJSON trace. Your coding agent can inspect what the framework discovered, run the agent, and verify the result without reading terminal prose.</p><h2 id="how-do-i-create-an-agent-with-the-built-in-skill" tabindex="-1">How do I create an agent with the built-in skill? <a class="header-anchor" href="#how-do-i-create-an-agent-with-the-built-in-skill" aria-label="Permalink to "How do I create an agent with the built-in skill?""></a></h2><p>Have the coding agent read <a href="./../skills/create-agent/SKILL.html"><code>skills/create-agent/SKILL.md</code></a> (in the monorepo: <code>packages/agent-serve/skills/create-agent/SKILL.md</code>) and follow it.</p><p>The skill asks about your agent's purpose, runtime, model, channels, MCP connections, and capabilities. It then shows you a plan, writes the project, and verifies the result.</p><p>For example:</p><blockquote><p>Use the Agent SDK create-agent skill to build a PR triage agent reachable through GitHub. It should summarize failed checks, require approval before posting a review, and include one smoke eval.</p></blockquote><p>The skill starts with a small surface: short instructions, one or two tools, the channels you chose, and a smoke eval. You can add more after the first end-to-end turn works.</p><h2 id="which-built-in-skill-should-i-use" tabindex="-1">Which built-in skill should I use? <a class="header-anchor" href="#which-built-in-skill-should-i-use" aria-label="Permalink to "Which built-in skill should I use?""></a></h2><p>The package ships task-specific guides under <a href="./../skills/"><code>skills/</code></a>:</p><table tabindex="0"><thead><tr><th>What you want to do</th><th>Skill</th></tr></thead><tbody><tr><td>Understand the project layout and runtimes</td><td><a href="./../skills/framework-map/SKILL.html"><code>framework-map</code></a></td></tr><tr><td>Create and verify a new agent</td><td><a href="./../skills/create-agent/SKILL.html"><code>create-agent</code></a></td></tr><tr><td>Write fixtures and regression checks</td><td><a href="./../skills/evals/SKILL.html"><code>evals</code></a></td></tr><tr><td>Live A/B metrics on traffic (<code>defineAB</code>)</td><td><a href="./../skills/ab/SKILL.html"><code>ab</code></a></td></tr><tr><td>Improve an agent against fixed inputs</td><td><a href="./../skills/hillclimb/SKILL.html"><code>hillclimb</code></a></td></tr><tr><td>Add GitHub webhooks and replay events</td><td><a href="./../skills/github/SKILL.html"><code>github</code></a></td></tr><tr><td>Connect an agent to Slack</td><td><a href="./../skills/setup-slack/SKILL.html"><code>setup-slack</code></a></td></tr><tr><td>Diagnose a local run</td><td><a href="./../skills/debug/SKILL.html"><code>debug</code></a></td></tr></tbody></table><p>Point your coding agent at the matching <code>SKILL.md</code>. The guide contains the workflow, commands, and common mistakes for that task.</p><div class="note custom-block github-alert"><p class="custom-block-title">NOTE</p><p>The skill bodies use the current <code>agent-serve</code> CLI names. This guide uses the upcoming <code>agent-sdk</code> names. See <a href="/docs/#run-the-cli">Run the CLI</a> for the full rename table.</p></div><h2 id="how-does-a-coding-agent-verify-its-work" tabindex="-1">How does a coding agent verify its work? <a class="header-anchor" href="#how-does-a-coding-agent-verify-its-work" aria-label="Permalink to "How does a coding agent verify its work?""></a></h2><p>The coding agent should discover the project, test each server tool, run a real turn, and finish with evals:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> validate</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span></span>
|
|
22
24
|
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> info</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span>
|
|
23
25
|
<span class="line"></span>
|
|
24
26
|
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> call</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> inspect_pr</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
|
|
@@ -31,7 +33,7 @@
|
|
|
31
33
|
<span class="line"></span>
|
|
32
34
|
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span></span>
|
|
33
35
|
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p><code>serve</code> boots without an API key, which is enough to check channel mounts and the playground shell. Model turns need a credential. When the environment lacks one, finish every key-free check, then hand the <code>run</code> and <code>eval</code> commands to the operator.</p><p>Test server tools with <code>call</code> before tuning the prompt. It runs a tool in-process with schema validation and no model turn. If the tool returns the wrong data, a prompt change won't fix it.</p><p><code>validate</code> and <code>run</code> don't type-check the project because tsx strips types. Run the project's TypeScript check before shipping. Tool results must also be JSON-shaped. Use object literals or <code>type</code> aliases for <code>execute</code> return types instead of <code>interface</code> types.</p><h2 id="how-do-i-improve-the-agent-after-its-first-run" tabindex="-1">How do I improve the agent after its first run? <a class="header-anchor" href="#how-do-i-improve-the-agent-after-its-first-run" aria-label="Permalink to "How do I improve the agent after its first run?""></a></h2><p>Scaffolding proves the agent runs. It says nothing about quality. Once a smoke turn passes, give the hillclimb skill:</p><ol><li><strong>Fixtures</strong>: one to three fixed inputs, such as a PR URL, a canonical question, or a saved webhook payload</li><li><strong>Success criteria</strong>: correct tool choice, fewer tool calls, lower wall time, or better output</li><li><strong>The freeze line</strong>: the API contract, output shape, and existing evals that must stay unchanged</li></ol><p>Have the coding agent read <a href="./../skills/hillclimb/SKILL.html"><code>skills/hillclimb/SKILL.md</code></a>. It measures the current run, proposes one change, remeasures the same fixtures, and adds an eval for each kept improvement.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to "Related""></a></h2><ul><li><a href="./quickstart.html">Create your first agent</a></li><li><a href="./scaffolding-agents.html">Scaffold an agent with Cursor</a></li><li><a href="./evals.html">Evals</a></li><li><a href="./hillclimbing.html">Hillclimbing</a></li><li><a href="./reference/cli.html">CLI reference</a></li></ul></div></div></main><footer class="VPDocFooter" data-v-2e506a34 data-v-574f06fa><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-574f06fa><span class="visually-hidden" id="doc-footer-aria-label" data-v-574f06fa>Pager</span><div class="pager" data-v-574f06fa><a class="VPLink link pager-link prev" href="/docs/templates/triage.html" data-v-574f06fa><!--[--><span class="desc" data-v-574f06fa>Previous page</span><span class="title" data-v-574f06fa>Triage agent</span><!--]--></a></div><div class="pager" data-v-574f06fa><a class="VPLink link pager-link next" href="/docs/evals.html" data-v-574f06fa><!--[--><span class="desc" data-v-574f06fa>Next page</span><span class="title" data-v-574f06fa>Evals</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
34
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"ab.md\":\"CVzWxLoB\",\"building-with-agents.md\":\"DH8A_cHA\",\"concepts.md\":\"CRfU3bVg\",\"deployment.md\":\"DX_hc3ze\",\"evals.md\":\"a0SMN6r9\",\"example-agents_approval-buddy.md\":\"DNL83puR\",\"example-agents_benny.md\":\"C40vHRLc\",\"example-agents_bugbot.md\":\"BRGMi9O2\",\"example-agents_codebase-wiki.md\":\"Dftj_tPp\",\"example-agents_codeowners-review.md\":\"Bfta-lBU\",\"example-agents_concierge.md\":\"MrKpQndp\",\"example-agents_fsd.md\":\"ZWHWWZPE\",\"example-agents_index.md\":\"QZ8mhr6n\",\"example-agents_knowledge-base.md\":\"DqKqHQ9u\",\"example-agents_oncall.md\":\"DK4XkYTd\",\"example-agents_security-reviewer.md\":\"Bai6D0Ee\",\"example-agents_slack-agent.md\":\"D7Kdj5BV\",\"example-agents_weather-agent.md\":\"lVEAbWFf\",\"guides_agent-to-agent.md\":\"BCeVdJRJ\",\"guides_cloud-runtime.md\":\"BSMLIBHr\",\"guides_convert-automation.md\":\"D06eIzea\",\"guides_github.md\":\"Cdt1s2QC\",\"guides_human-in-the-loop.md\":\"By1G2T3_\",\"guides_mcp-oauth.md\":\"Du0f7pGU\",\"guides_opentelemetry.md\":\"bmPmkvJu\",\"guides_slack.md\":\"DiUmk_Oi\",\"guides_webhooks.md\":\"BpnIdO0i\",\"hillclimbing.md\":\"ywF3yDAd\",\"index.md\":\"BAaMXLFd\",\"quickstart.md\":\"DsrarzEg\",\"reference_agent-config.md\":\"Bqylgw50\",\"reference_artifacts.md\":\"Dior32Qw\",\"reference_channels.md\":\"DQZjCnyh\",\"reference_cli.md\":\"B7GkAJRC\",\"reference_connections.md\":\"DYidrb-j\",\"reference_hooks.md\":\"B9FSgdDe\",\"reference_http-api.md\":\"CSHVobzG\",\"reference_instructions.md\":\"DhNCOl7r\",\"reference_playground.md\":\"Dfb92yQf\",\"reference_project-layout.md\":\"CwkSbEWT\",\"reference_prompt.md\":\"DZUMtLPD\",\"reference_schedules.md\":\"DNipebiG\",\"reference_sessions.md\":\"tUFzz98S\",\"reference_skills.md\":\"B5ZEuHfG\",\"reference_subagents.md\":\"Xoav0AII\",\"reference_tools.md\":\"wpaJtHn6\",\"scaffolding-agents.md\":\"CRDDUtYJ\",\"storage.md\":\"JbjlHWZ6\",\"templates_agentic-owners.md\":\"DSJSIpWU\",\"templates_demo.md\":\"DhFcWN6j\",\"templates_pr-autofixer.md\":\"1HAR3RXE\",\"templates_security-reviewer.md\":\"ByFyRta2\",\"templates_triage.md\":\"CVlpctKS\",\"troubleshooting.md\":\"DYECCZiJ\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"Agent SDK\",\"description\":\"Filesystem-first framework for building and serving Cursor agents.\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"sdkVersion\":\"0.1.
|
|
36
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"ab.md\":\"CVzWxLoB\",\"building-with-agents.md\":\"DH8A_cHA\",\"concepts.md\":\"CRfU3bVg\",\"deployment.md\":\"DX_hc3ze\",\"evals.md\":\"a0SMN6r9\",\"example-agents_approval-buddy.md\":\"DNL83puR\",\"example-agents_benny.md\":\"C40vHRLc\",\"example-agents_bugbot.md\":\"BRGMi9O2\",\"example-agents_codebase-wiki.md\":\"Dftj_tPp\",\"example-agents_codeowners-review.md\":\"Bfta-lBU\",\"example-agents_concierge.md\":\"MrKpQndp\",\"example-agents_fsd.md\":\"ZWHWWZPE\",\"example-agents_index.md\":\"QZ8mhr6n\",\"example-agents_knowledge-base.md\":\"DqKqHQ9u\",\"example-agents_oncall.md\":\"DK4XkYTd\",\"example-agents_security-reviewer.md\":\"Bai6D0Ee\",\"example-agents_slack-agent.md\":\"D7Kdj5BV\",\"example-agents_weather-agent.md\":\"lVEAbWFf\",\"guides_agent-to-agent.md\":\"BCeVdJRJ\",\"guides_cloud-runtime.md\":\"BSMLIBHr\",\"guides_convert-automation.md\":\"D06eIzea\",\"guides_github.md\":\"Cdt1s2QC\",\"guides_human-in-the-loop.md\":\"By1G2T3_\",\"guides_mcp-oauth.md\":\"Du0f7pGU\",\"guides_opentelemetry.md\":\"bmPmkvJu\",\"guides_slack.md\":\"DiUmk_Oi\",\"guides_webhooks.md\":\"BpnIdO0i\",\"hillclimbing.md\":\"ywF3yDAd\",\"index.md\":\"BAaMXLFd\",\"quickstart.md\":\"DsrarzEg\",\"reference_agent-config.md\":\"Bqylgw50\",\"reference_artifacts.md\":\"Dior32Qw\",\"reference_channels.md\":\"DQZjCnyh\",\"reference_cli.md\":\"B7GkAJRC\",\"reference_connections.md\":\"DYidrb-j\",\"reference_hooks.md\":\"B9FSgdDe\",\"reference_http-api.md\":\"CSHVobzG\",\"reference_instructions.md\":\"DhNCOl7r\",\"reference_playground.md\":\"Dfb92yQf\",\"reference_project-layout.md\":\"CwkSbEWT\",\"reference_prompt.md\":\"DZUMtLPD\",\"reference_schedules.md\":\"DNipebiG\",\"reference_sessions.md\":\"tUFzz98S\",\"reference_skills.md\":\"B5ZEuHfG\",\"reference_subagents.md\":\"Xoav0AII\",\"reference_tools.md\":\"wpaJtHn6\",\"scaffolding-agents.md\":\"CRDDUtYJ\",\"storage.md\":\"JbjlHWZ6\",\"templates_agentic-owners.md\":\"DSJSIpWU\",\"templates_demo.md\":\"DhFcWN6j\",\"templates_pr-autofixer.md\":\"1HAR3RXE\",\"templates_security-reviewer.md\":\"ByFyRta2\",\"templates_triage.md\":\"CVlpctKS\",\"troubleshooting.md\":\"DYECCZiJ\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"Agent SDK\",\"description\":\"Filesystem-first framework for building and serving Cursor agents.\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"sdkVersion\":\"0.1.92\",\"nav\":[{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Guides\",\"link\":\"/guides/webhooks\"},{\"text\":\"Examples\",\"link\":\"/example-agents/\"},{\"text\":\"Reference\",\"link\":\"/reference/project-layout\"}],\"search\":{\"provider\":\"local\"},\"outline\":{\"level\":[2,3]},\"sidebar\":[{\"text\":\"Getting started\",\"items\":[{\"text\":\"Overview\",\"link\":\"/\"},{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Scaffold an agent with Cursor\",\"link\":\"/scaffolding-agents\"},{\"text\":\"Convert a Cursor Automation\",\"link\":\"/guides/convert-automation\"},{\"text\":\"Concepts\",\"link\":\"/concepts\"}]},{\"text\":\"Templates\",\"items\":[{\"text\":\"Demo agent\",\"link\":\"/templates/demo\"},{\"text\":\"Security reviewer\",\"link\":\"/templates/security-reviewer\"},{\"text\":\"Agentic Owners\",\"link\":\"/templates/agentic-owners\"},{\"text\":\"PR autofixer\",\"link\":\"/templates/pr-autofixer\"},{\"text\":\"Triage agent\",\"link\":\"/templates/triage\"}]},{\"text\":\"Improving agents\",\"items\":[{\"text\":\"Building agents with agents\",\"link\":\"/building-with-agents\"},{\"text\":\"Evals\",\"link\":\"/evals\"},{\"text\":\"Live A/B metrics\",\"link\":\"/ab\"},{\"text\":\"Storage\",\"link\":\"/storage\"},{\"text\":\"Hillclimbing\",\"link\":\"/hillclimbing\"}]},{\"text\":\"Guides\",\"items\":[{\"text\":\"Webhooks & custom channels\",\"link\":\"/guides/webhooks\"},{\"text\":\"GitHub\",\"link\":\"/guides/github\"},{\"text\":\"Slack\",\"link\":\"/guides/slack\"},{\"text\":\"Human-in-the-loop approvals\",\"link\":\"/guides/human-in-the-loop\"},{\"text\":\"Agent-to-agent\",\"link\":\"/guides/agent-to-agent\"},{\"text\":\"Host MCP OAuth\",\"link\":\"/guides/mcp-oauth\"},{\"text\":\"Cloud runtime\",\"link\":\"/guides/cloud-runtime\"},{\"text\":\"OpenTelemetry\",\"link\":\"/guides/opentelemetry\"}]},{\"text\":\"Example agents\",\"items\":[{\"text\":\"Choose an example\",\"link\":\"/example-agents/\"},{\"text\":\"Weather agent\",\"link\":\"/example-agents/weather-agent\"},{\"text\":\"Slack agent\",\"link\":\"/example-agents/slack-agent\"},{\"text\":\"Concierge\",\"link\":\"/example-agents/concierge\"},{\"text\":\"Playbook router\",\"link\":\"/example-agents/benny\"},{\"text\":\"Alert investigator\",\"link\":\"/example-agents/oncall\"},{\"text\":\"PR evidence reviewer\",\"link\":\"/example-agents/bugbot\"},{\"text\":\"Approval Buddy\",\"link\":\"/example-agents/approval-buddy\"},{\"text\":\"Security Reviewer\",\"link\":\"/example-agents/security-reviewer\"},{\"text\":\"Remote PR coordinator\",\"link\":\"/example-agents/fsd\"},{\"text\":\"Knowledge base\",\"link\":\"/example-agents/knowledge-base\"},{\"text\":\"Codebase wiki\",\"link\":\"/example-agents/codebase-wiki\"},{\"text\":\"Codeowners review\",\"link\":\"/example-agents/codeowners-review\"}]},{\"text\":\"Operating\",\"items\":[{\"text\":\"Deployment\",\"link\":\"/deployment\"},{\"text\":\"Fix common problems\",\"link\":\"/troubleshooting\"}]},{\"text\":\"Reference\",\"collapsed\":false,\"items\":[{\"text\":\"Project layout\",\"link\":\"/reference/project-layout\"},{\"text\":\"Agent config\",\"link\":\"/reference/agent-config\"},{\"text\":\"Instructions\",\"link\":\"/reference/instructions\"},{\"text\":\"Tools\",\"link\":\"/reference/tools\"},{\"text\":\"prompt\",\"link\":\"/reference/prompt\"},{\"text\":\"Skills\",\"link\":\"/reference/skills\"},{\"text\":\"MCP Connections\",\"link\":\"/reference/connections\"},{\"text\":\"Subagents\",\"link\":\"/reference/subagents\"},{\"text\":\"Channels\",\"link\":\"/reference/channels\"},{\"text\":\"Schedules & reminders\",\"link\":\"/reference/schedules\"},{\"text\":\"Hooks\",\"link\":\"/reference/hooks\"},{\"text\":\"Artifacts\",\"link\":\"/reference/artifacts\"},{\"text\":\"Sessions & streaming\",\"link\":\"/reference/sessions\"},{\"text\":\"Playground\",\"link\":\"/reference/playground\"},{\"text\":\"CLI\",\"link\":\"/reference/cli\"},{\"text\":\"HTTP API\",\"link\":\"/reference/http-api\"}]}]},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
|
|
35
37
|
|
|
36
38
|
</body>
|
|
37
39
|
</html>
|