@cursor/july 0.1.111 → 0.1.113
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/bin/agent-serve.js +20 -4
- package/dist/channels/checks.d.ts +10 -0
- package/dist/channels/checks.d.ts.map +1 -1
- package/dist/channels/github/github-channel.d.ts +29 -19
- package/dist/channels/github/github-channel.d.ts.map +1 -1
- package/dist/channels/github/github-channel.js +29 -19
- package/dist/channels/origin/checks.d.ts +1 -1
- package/dist/channels/origin/checks.d.ts.map +1 -1
- package/dist/channels/slack/agentic-delivery.d.ts.map +1 -1
- package/dist/channels/slack/agentic-delivery.js +2 -0
- package/dist/channels/slack/dispatch.d.ts +11 -1
- package/dist/channels/slack/dispatch.d.ts.map +1 -1
- package/dist/channels/slack/dispatch.js +146 -39
- package/dist/channels/slack/placeholder.d.ts +26 -0
- package/dist/channels/slack/placeholder.d.ts.map +1 -1
- package/dist/channels/slack/placeholder.js +28 -0
- package/dist/channels/slack/slack-channel.d.ts.map +1 -1
- package/dist/channels/slack/slack-channel.js +9 -16
- package/dist/docs/404.html +2 -2
- package/dist/docs/assets/{app.Drol6mi6.js → app.CAeK13eM.js} +4 -4
- package/dist/docs/assets/building-with-agents.md.BBCx0AUo.js +9 -0
- package/dist/docs/assets/building-with-agents.md.BBCx0AUo.lean.js +1 -0
- package/dist/docs/assets/chunks/@localSearchIndexroot.Ck9E52Ls.js +1 -0
- package/dist/docs/assets/chunks/{VPLocalSearchBox.CdqoBZJX.js → VPLocalSearchBox.C9LbPHod.js} +1 -1
- package/dist/docs/assets/chunks/{arc.BVX3ycTn.js → arc.CmMq2zmS.js} +1 -1
- package/dist/docs/assets/chunks/{architectureDiagram-Q4EWVU46.CcChnMxX.js → architectureDiagram-Q4EWVU46.CCXB8Uj5.js} +1 -1
- package/dist/docs/assets/chunks/{baseUniq.xwtXO-yt.js → baseUniq.CyQo6eLe.js} +1 -1
- package/dist/docs/assets/chunks/{blockDiagram-DXYQGD6D.CijZ_taK.js → blockDiagram-DXYQGD6D.JYq6w91N.js} +1 -1
- package/dist/docs/assets/chunks/{c4Diagram-AHTNJAMY.0FBvwBvK.js → c4Diagram-AHTNJAMY.BRV8GPJJ.js} +1 -1
- package/dist/docs/assets/chunks/channel.BHiYmnZ4.js +1 -0
- package/dist/docs/assets/chunks/{chunk-4BX2VUAB.CELMmMDA.js → chunk-4BX2VUAB.Bv4ooYQR.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-4TB4RGXK.C4bhFtSm.js → chunk-4TB4RGXK.t4JtKPcj.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-55IACEB6.ZnQ9gRRQ.js → chunk-55IACEB6.34lCHj9Y.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-EDXVE4YY.DPlmglG-.js → chunk-EDXVE4YY.BSwrPNrt.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-FMBD7UC4.BrUk8pff.js → chunk-FMBD7UC4.Beeun-R-.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-OYMX7WX6.CcmWIncu.js → chunk-OYMX7WX6.BUUFUcJc.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-QZHKN3VN.D65-cs8I.js → chunk-QZHKN3VN.B2XjHzN_.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-YZCP3GAM.qoXZpG9F.js → chunk-YZCP3GAM.CLYG8znk.js} +1 -1
- package/dist/docs/assets/chunks/classDiagram-6PBFFD2Q.Degh8l90.js +1 -0
- package/dist/docs/assets/chunks/classDiagram-v2-HSJHXN6E.Degh8l90.js +1 -0
- package/dist/docs/assets/chunks/clone.BIywbczV.js +1 -0
- package/dist/docs/assets/chunks/{cose-bilkent-S5V4N54A.8rYtqudO.js → cose-bilkent-S5V4N54A.DVEa6fZp.js} +1 -1
- package/dist/docs/assets/chunks/{dagre-KV5264BT.DrRP1fOh.js → dagre-KV5264BT.C9PZQK-S.js} +1 -1
- package/dist/docs/assets/chunks/{diagram-5BDNPKRD.DHj_xA2_.js → diagram-5BDNPKRD.DoN0uv3Y.js} +1 -1
- package/dist/docs/assets/chunks/{diagram-G4DWMVQ6.Bz_6nAKj.js → diagram-G4DWMVQ6.Czv3duqx.js} +1 -1
- package/dist/docs/assets/chunks/{diagram-MMDJMWI5.BWA0xSW9.js → diagram-MMDJMWI5.BinJ5kWb.js} +1 -1
- package/dist/docs/assets/chunks/{diagram-TYMM5635.CpTJLNJI.js → diagram-TYMM5635.DW326M4K.js} +1 -1
- package/dist/docs/assets/chunks/{erDiagram-SMLLAGMA.-7AWWSrP.js → erDiagram-SMLLAGMA.U2pR_OA7.js} +1 -1
- package/dist/docs/assets/chunks/{flowDiagram-DWJPFMVM.BTnQ742_.js → flowDiagram-DWJPFMVM.ByWJXeYK.js} +1 -1
- package/dist/docs/assets/chunks/framework.BNw1pucY.js +19 -0
- package/dist/docs/assets/chunks/{ganttDiagram-T4ZO3ILL.B5_HiiQ5.js → ganttDiagram-T4ZO3ILL.OquF0Rtg.js} +1 -1
- package/dist/docs/assets/chunks/{gitGraphDiagram-UUTBAWPF.CZcNTFZd.js → gitGraphDiagram-UUTBAWPF.Bpn01P7X.js} +1 -1
- package/dist/docs/assets/chunks/{graph.V2GLaab4.js → graph.CNRB6ETL.js} +1 -1
- package/dist/docs/assets/chunks/{infoDiagram-42DDH7IO.7pZOkCCU.js → infoDiagram-42DDH7IO.CqhknMWi.js} +1 -1
- package/dist/docs/assets/chunks/{ishikawaDiagram-UXIWVN3A.DSMo3Qa3.js → ishikawaDiagram-UXIWVN3A.C6xpR2af.js} +1 -1
- package/dist/docs/assets/chunks/{journeyDiagram-VCZTEJTY.BNEgWN1S.js → journeyDiagram-VCZTEJTY.Cg5f7oB3.js} +1 -1
- package/dist/docs/assets/chunks/{kanban-definition-6JOO6SKY.B-4d4tC7.js → kanban-definition-6JOO6SKY.Cx9YTwlU.js} +1 -1
- package/dist/docs/assets/chunks/{layout.Dvbn9nSb.js → layout.ljS-wFtK.js} +1 -1
- package/dist/docs/assets/chunks/{linear.D2GM4p4b.js → linear.jSxNrsFC.js} +1 -1
- package/dist/docs/assets/chunks/{min.DVBtLK-B.js → min.Cum8AlQw.js} +1 -1
- package/dist/docs/assets/chunks/{mindmap-definition-QFDTVHPH.CnJRI55x.js → mindmap-definition-QFDTVHPH.BLiysLpe.js} +1 -1
- package/dist/docs/assets/chunks/{pieDiagram-DEJITSTG.CtoaTFlA.js → pieDiagram-DEJITSTG.BoIDyuKF.js} +1 -1
- package/dist/docs/assets/chunks/{quadrantDiagram-34T5L4WZ.DtT24_vN.js → quadrantDiagram-34T5L4WZ.DLkpDytR.js} +1 -1
- package/dist/docs/assets/chunks/{requirementDiagram-MS252O5E.CjSO4o8f.js → requirementDiagram-MS252O5E.DqTVqSu2.js} +1 -1
- package/dist/docs/assets/chunks/{sankeyDiagram-XADWPNL6.-wWiIVWa.js → sankeyDiagram-XADWPNL6.CG_6FF7j.js} +1 -1
- package/dist/docs/assets/chunks/{sequenceDiagram-FGHM5R23.DSk8s4gX.js → sequenceDiagram-FGHM5R23.BIp9602K.js} +1 -1
- package/dist/docs/assets/chunks/{stateDiagram-FHFEXIEX.BFKAsdkN.js → stateDiagram-FHFEXIEX.COSXsD9I.js} +1 -1
- package/dist/docs/assets/chunks/stateDiagram-v2-QKLJ7IA2.qrxrbFsX.js +1 -0
- package/dist/docs/assets/chunks/{theme.C0MctGaz.js → theme.CXJ7PNwy.js} +2 -2
- package/dist/docs/assets/chunks/{timeline-definition-GMOUNBTQ.TUNJbAFe.js → timeline-definition-GMOUNBTQ.CXdVqkLq.js} +1 -1
- package/dist/docs/assets/chunks/{vennDiagram-DHZGUBPP.C-RGNnk4.js → vennDiagram-DHZGUBPP.CZxGuc4r.js} +1 -1
- package/dist/docs/assets/chunks/{wardley-RL74JXVD.g5efOWmT.js → wardley-RL74JXVD.3oVgfqQk.js} +1 -1
- package/dist/docs/assets/chunks/{wardleyDiagram-NUSXRM2D.DH4zt73Y.js → wardleyDiagram-NUSXRM2D.6_irCgGJ.js} +1 -1
- package/dist/docs/assets/chunks/{xychartDiagram-5P7HB3ND.B-3k6bF6.js → xychartDiagram-5P7HB3ND.TRPe92m3.js} +1 -1
- package/dist/docs/assets/deployment.md.D2jQZuFx.js +32 -0
- package/dist/docs/assets/deployment.md.D2jQZuFx.lean.js +1 -0
- package/dist/docs/assets/evals.md.D3Y3Aixt.js +72 -0
- package/dist/docs/assets/evals.md.D3Y3Aixt.lean.js +1 -0
- package/dist/docs/assets/guides_agent-to-agent.md.CD4T5FIl.js +41 -0
- package/dist/docs/assets/guides_agent-to-agent.md.CD4T5FIl.lean.js +1 -0
- package/dist/docs/assets/guides_bitbucket.md.mpevW-VP.js +145 -0
- package/dist/docs/assets/guides_bitbucket.md.mpevW-VP.lean.js +1 -0
- package/dist/docs/assets/guides_cloud-agents.md.Cp1O3u-X.js +15 -0
- package/dist/docs/assets/guides_cloud-agents.md.Cp1O3u-X.lean.js +1 -0
- package/dist/docs/assets/guides_convert-automation.md.CqEyfP6Y.js +43 -0
- package/dist/docs/assets/guides_convert-automation.md.CqEyfP6Y.lean.js +1 -0
- package/dist/docs/assets/guides_github.md.BwpBp3ed.js +156 -0
- package/dist/docs/assets/guides_github.md.BwpBp3ed.lean.js +1 -0
- package/dist/docs/assets/guides_gitlab.md.DaEC3nMk.js +153 -0
- package/dist/docs/assets/guides_gitlab.md.DaEC3nMk.lean.js +1 -0
- package/dist/docs/assets/guides_grokbot-agents.md.CMhZNdEU.js +16 -0
- package/dist/docs/assets/guides_grokbot-agents.md.CMhZNdEU.lean.js +1 -0
- package/dist/docs/assets/guides_improve.md.Bnp4F99w.js +22 -0
- package/dist/docs/assets/guides_improve.md.Bnp4F99w.lean.js +1 -0
- package/dist/docs/assets/guides_jev.md.F5fAkkfN.js +189 -0
- package/dist/docs/assets/guides_jev.md.F5fAkkfN.lean.js +1 -0
- package/dist/docs/assets/guides_mcp-oauth.md.bSFakfCY.js +50 -0
- package/dist/docs/assets/guides_mcp-oauth.md.bSFakfCY.lean.js +1 -0
- package/dist/docs/assets/guides_opentelemetry.md.BKDxQmmd.js +35 -0
- package/dist/docs/assets/guides_opentelemetry.md.BKDxQmmd.lean.js +1 -0
- package/dist/docs/assets/guides_slack.md.Bo96y42E.js +70 -0
- package/dist/docs/assets/guides_slack.md.Bo96y42E.lean.js +1 -0
- package/dist/docs/assets/guides_webhooks.md.1A72_VEE.js +92 -0
- package/dist/docs/assets/guides_webhooks.md.1A72_VEE.lean.js +1 -0
- package/dist/docs/assets/hillclimbing.md.D4E1o5Sa.js +7 -0
- package/dist/docs/assets/hillclimbing.md.D4E1o5Sa.lean.js +1 -0
- package/dist/docs/assets/{index.md.BFVyY2KT.js → index.md.C-t81M5J.js} +2 -2
- package/dist/docs/assets/{index.md.BFVyY2KT.lean.js → index.md.C-t81M5J.lean.js} +1 -1
- package/dist/docs/assets/{quickstart.md.D3MjSZN-.js → quickstart.md.DAvVhuuU.js} +1 -1
- package/dist/docs/assets/{quickstart.md.D3MjSZN-.lean.js → quickstart.md.DAvVhuuU.lean.js} +1 -1
- package/dist/docs/assets/{reference_agent-config.md.CfVA-LZJ.js → reference_agent-config.md.DGPyw7ms.js} +1 -1
- package/dist/docs/assets/{reference_agent-config.md.CfVA-LZJ.lean.js → reference_agent-config.md.DGPyw7ms.lean.js} +1 -1
- package/dist/docs/assets/{reference_artifacts.md.Vf7qyIZ-.js → reference_artifacts.md.Bu_4HmsD.js} +1 -1
- package/dist/docs/assets/{reference_artifacts.md.Vf7qyIZ-.lean.js → reference_artifacts.md.Bu_4HmsD.lean.js} +1 -1
- package/dist/docs/assets/{reference_channels.md.icqLKcTc.js → reference_channels.md.nFWbzAic.js} +1 -1
- package/dist/docs/assets/{reference_channels.md.icqLKcTc.lean.js → reference_channels.md.nFWbzAic.lean.js} +1 -1
- package/dist/docs/assets/{reference_cli.md.B2dBL6L8.js → reference_cli.md.DLWDz9ij.js} +3 -1
- package/dist/docs/assets/{reference_cli.md.B2dBL6L8.lean.js → reference_cli.md.DLWDz9ij.lean.js} +1 -1
- package/dist/docs/assets/{reference_connections.md.Cb3U_c8n.js → reference_connections.md.Je9dMsdd.js} +2 -2
- package/dist/docs/assets/{reference_connections.md.Cb3U_c8n.lean.js → reference_connections.md.Je9dMsdd.lean.js} +1 -1
- package/dist/docs/assets/reference_evals.md.DNJzM_yf.js +57 -0
- package/dist/docs/assets/reference_evals.md.DNJzM_yf.lean.js +1 -0
- package/dist/docs/assets/reference_extensions.md.Cv5aLCz_.js +58 -0
- package/dist/docs/assets/reference_extensions.md.Cv5aLCz_.lean.js +1 -0
- package/dist/docs/assets/{reference_hooks.md.CZuynAxj.js → reference_hooks.md.B7uzNENk.js} +2 -2
- package/dist/docs/assets/{reference_hooks.md.CZuynAxj.lean.js → reference_hooks.md.B7uzNENk.lean.js} +1 -1
- package/dist/docs/assets/{reference_http-api.md.DTKcYE6L.js → reference_http-api.md.CduHavZ2.js} +1 -1
- package/dist/docs/assets/{reference_http-api.md.DTKcYE6L.lean.js → reference_http-api.md.CduHavZ2.lean.js} +1 -1
- package/dist/docs/assets/{reference_instructions.md.D7gkckK-.js → reference_instructions.md.CU1My5My.js} +1 -1
- package/dist/docs/assets/{reference_instructions.md.D7gkckK-.lean.js → reference_instructions.md.CU1My5My.lean.js} +1 -1
- package/dist/docs/assets/{reference_playground.md.D2YExv5K.js → reference_playground.md.Ch2d0Iqi.js} +1 -1
- package/dist/docs/assets/{reference_playground.md.D2YExv5K.lean.js → reference_playground.md.Ch2d0Iqi.lean.js} +1 -1
- package/dist/docs/assets/{reference_project-layout.md.DPxbUJyt.js → reference_project-layout.md.BGhgpy9V.js} +1 -1
- package/dist/docs/assets/{reference_project-layout.md.DPxbUJyt.lean.js → reference_project-layout.md.BGhgpy9V.lean.js} +1 -1
- package/dist/docs/assets/{reference_prompt.md.BQ5uAv1F.js → reference_prompt.md.Ccp0R53H.js} +1 -1
- package/dist/docs/assets/{reference_prompt.md.BQ5uAv1F.lean.js → reference_prompt.md.Ccp0R53H.lean.js} +1 -1
- package/dist/docs/assets/{reference_schedules.md.BasfZWO-.js → reference_schedules.md.B2Nm6FaD.js} +1 -1
- package/dist/docs/assets/{reference_schedules.md.BasfZWO-.lean.js → reference_schedules.md.B2Nm6FaD.lean.js} +1 -1
- package/dist/docs/assets/{reference_sessions.md.YKvIsWAx.js → reference_sessions.md.1_6Vyv7x.js} +1 -1
- package/dist/docs/assets/{reference_sessions.md.YKvIsWAx.lean.js → reference_sessions.md.1_6Vyv7x.lean.js} +1 -1
- package/dist/docs/assets/{reference_skills.md.rNgpsGd0.js → reference_skills.md.DjQkRefx.js} +1 -1
- package/dist/docs/assets/{reference_skills.md.rNgpsGd0.lean.js → reference_skills.md.DjQkRefx.lean.js} +1 -1
- package/dist/docs/assets/{reference_subagents.md.e5qitjJt.js → reference_subagents.md.Dl16gcBj.js} +2 -2
- package/dist/docs/assets/{reference_subagents.md.e5qitjJt.lean.js → reference_subagents.md.Dl16gcBj.lean.js} +1 -1
- package/dist/docs/assets/{reference_tools.md.BdCO2aHZ.js → reference_tools.md.B1dH1lpa.js} +3 -3
- package/dist/docs/assets/{reference_tools.md.BdCO2aHZ.lean.js → reference_tools.md.B1dH1lpa.lean.js} +1 -1
- package/dist/docs/assets/{templates_agentic-owners.md.Da_AGDlH.js → templates_agentic-owners.md.9M575F5C.js} +1 -1
- package/dist/docs/assets/{templates_agentic-owners.md.Da_AGDlH.lean.js → templates_agentic-owners.md.9M575F5C.lean.js} +1 -1
- package/dist/docs/assets/{templates_pr-autofixer.md.DqxocIGh.js → templates_pr-autofixer.md.ws0DDXDy.js} +1 -1
- package/dist/docs/assets/{templates_pr-autofixer.md.DqxocIGh.lean.js → templates_pr-autofixer.md.ws0DDXDy.lean.js} +1 -1
- package/dist/docs/assets/{templates_security-reviewer.md.Bhnvd8VE.js → templates_security-reviewer.md.KEFYXzfK.js} +1 -1
- package/dist/docs/assets/{templates_security-reviewer.md.Bhnvd8VE.lean.js → templates_security-reviewer.md.KEFYXzfK.lean.js} +1 -1
- package/dist/docs/assets/templates_thermo-quality-review.md.VNJ_mohX.js +3 -0
- package/dist/docs/assets/templates_thermo-quality-review.md.VNJ_mohX.lean.js +1 -0
- package/dist/docs/assets/templates_thermo-review.md.Hi3zWOkP.js +3 -0
- package/dist/docs/assets/templates_thermo-review.md.Hi3zWOkP.lean.js +1 -0
- package/dist/docs/assets/{templates_triage.md.BdBWO9Ic.js → templates_triage.md.CVGe_FG6.js} +1 -1
- package/dist/docs/assets/{templates_triage.md.BdBWO9Ic.lean.js → templates_triage.md.CVGe_FG6.lean.js} +1 -1
- package/dist/docs/assets/troubleshooting.md.mnfFG2Em.js +1 -0
- package/dist/docs/assets/troubleshooting.md.mnfFG2Em.lean.js +1 -0
- package/dist/docs/building-with-agents.html +44 -48
- package/dist/docs/building-with-agents.md +94 -82
- package/dist/docs/deployment.html +59 -78
- package/dist/docs/deployment.md +117 -363
- package/dist/docs/evals.html +85 -224
- package/dist/docs/evals.md +149 -673
- package/dist/docs/guides/agent-to-agent.html +74 -47
- package/dist/docs/guides/agent-to-agent.md +128 -46
- package/dist/docs/guides/bitbucket.html +179 -44
- package/dist/docs/guides/bitbucket.md +249 -48
- package/dist/docs/guides/cloud-agents.html +46 -40
- package/dist/docs/guides/cloud-agents.md +119 -66
- package/dist/docs/guides/convert-automation.html +80 -49
- package/dist/docs/guides/convert-automation.md +156 -147
- package/dist/docs/guides/github.html +184 -92
- package/dist/docs/guides/github.md +260 -245
- package/dist/docs/guides/gitlab.html +184 -45
- package/dist/docs/guides/gitlab.md +249 -50
- package/dist/docs/guides/grokbot-agents.html +48 -41
- package/dist/docs/guides/grokbot-agents.md +99 -53
- package/dist/docs/guides/improve.html +48 -40
- package/dist/docs/guides/improve.md +111 -58
- package/dist/docs/guides/jev.html +248 -0
- package/dist/docs/guides/jev.md +348 -0
- package/dist/docs/guides/mcp-oauth.html +74 -52
- package/dist/docs/guides/mcp-oauth.md +111 -121
- package/dist/docs/guides/opentelemetry.html +63 -54
- package/dist/docs/guides/opentelemetry.md +96 -165
- package/dist/docs/guides/slack.html +83 -59
- package/dist/docs/guides/slack.md +157 -227
- package/dist/docs/guides/webhooks.html +97 -230
- package/dist/docs/guides/webhooks.md +154 -385
- package/dist/docs/hashmap.json +1 -1
- package/dist/docs/hillclimbing.html +44 -38
- package/dist/docs/hillclimbing.md +101 -55
- package/dist/docs/index.html +38 -38
- package/dist/docs/index.md +2 -0
- package/dist/docs/llms-full.txt +3490 -3146
- package/dist/docs/llms.txt +22 -18
- package/dist/docs/quickstart.html +37 -37
- package/dist/docs/reference/agent-config.html +37 -37
- package/dist/docs/reference/artifacts.html +38 -38
- package/dist/docs/reference/channels.html +37 -37
- package/dist/docs/reference/cli.html +39 -37
- package/dist/docs/reference/cli.md +2 -0
- package/dist/docs/reference/connections.html +38 -38
- package/dist/docs/reference/connections.md +1 -1
- package/dist/docs/reference/evals.html +116 -0
- package/dist/docs/reference/evals.md +293 -0
- package/dist/docs/reference/extensions.html +80 -84
- package/dist/docs/reference/extensions.md +131 -202
- package/dist/docs/reference/hooks.html +39 -39
- package/dist/docs/reference/hooks.md +34 -34
- package/dist/docs/reference/http-api.html +37 -37
- package/dist/docs/reference/instructions.html +37 -37
- package/dist/docs/reference/playground.html +37 -37
- package/dist/docs/reference/project-layout.html +37 -37
- package/dist/docs/reference/prompt.html +37 -37
- package/dist/docs/reference/schedules.html +37 -37
- package/dist/docs/reference/sessions.html +37 -37
- package/dist/docs/reference/skills.html +37 -37
- package/dist/docs/reference/subagents.html +38 -38
- package/dist/docs/reference/subagents.md +1 -1
- package/dist/docs/reference/tools.html +38 -38
- package/dist/docs/reference/tools.md +4 -0
- package/dist/docs/templates/agentic-owners.html +38 -38
- package/dist/docs/templates/pr-autofixer.html +37 -37
- package/dist/docs/templates/security-reviewer.html +38 -38
- package/dist/docs/templates/thermo-quality-review.html +62 -0
- package/dist/docs/templates/thermo-quality-review.md +75 -0
- package/dist/docs/templates/thermo-review.html +62 -0
- package/dist/docs/templates/thermo-review.md +74 -0
- package/dist/docs/templates/triage.html +37 -37
- package/dist/docs/troubleshooting.html +38 -38
- package/dist/docs/troubleshooting.md +97 -66
- package/dist/extensions/cursor-cloud-agents/skills/handoff.md +3 -7
- package/dist/extensions/improve/extension.d.ts +3 -1
- package/dist/extensions/improve/extension.d.ts.map +1 -1
- package/dist/extensions/improve/extension.js +4 -2
- package/dist/extensions/improve/skills/yourself.js +1 -1
- package/dist/extensions/jev/extension.d.ts +43 -0
- package/dist/extensions/jev/extension.d.ts.map +1 -0
- package/dist/extensions/jev/extension.js +47 -0
- package/dist/extensions/jev/lib/evaluate.d.ts +101 -0
- package/dist/extensions/jev/lib/evaluate.d.ts.map +1 -0
- package/dist/extensions/jev/lib/evaluate.js +167 -0
- package/dist/extensions/jev/skills/gated-write.md +25 -0
- package/dist/extensions/jev/skills/questions.md +33 -0
- package/dist/extensions/jev/tools/evaluate.d.ts +4 -0
- package/dist/extensions/jev/tools/evaluate.d.ts.map +1 -0
- package/dist/extensions/jev/tools/evaluate.js +88 -0
- package/dist/extensions.d.ts +1 -1
- package/dist/extensions.d.ts.map +1 -1
- package/dist/extensions.js +2 -0
- package/dist/filesystem.d.ts +46 -2
- package/dist/filesystem.d.ts.map +1 -1
- package/dist/filesystem.js +149 -102
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/internal/advertise-tools.d.ts.map +1 -1
- package/dist/internal/advertise-tools.js +6 -0
- package/dist/internal/cli-ax.d.ts +6 -0
- package/dist/internal/cli-ax.d.ts.map +1 -1
- package/dist/internal/cli-ax.js +72 -14
- package/dist/internal/continuation-identity.d.ts.map +1 -1
- package/dist/internal/continuation-identity.js +1 -0
- package/dist/internal/cursor-agent-template.d.ts +1 -1
- package/dist/internal/cursor-agent-template.d.ts.map +1 -1
- package/dist/internal/cursor-agent-template.js +2 -0
- package/dist/internal/discovery/connections.d.ts.map +1 -1
- package/dist/internal/discovery/connections.js +18 -0
- package/dist/internal/discovery/extensions.d.ts.map +1 -1
- package/dist/internal/discovery/extensions.js +8 -4
- package/dist/internal/discovery/info.d.ts.map +1 -1
- package/dist/internal/discovery/info.js +1 -0
- package/dist/internal/filesystem/tools.d.ts.map +1 -1
- package/dist/internal/filesystem/tools.js +2 -2
- package/dist/internal/filesystem/walk.d.ts +7 -4
- package/dist/internal/filesystem/walk.d.ts.map +1 -1
- package/dist/internal/filesystem/walk.js +34 -11
- package/dist/internal/hosted-admission-adapter.d.ts +3 -0
- package/dist/internal/hosted-admission-adapter.d.ts.map +1 -1
- package/dist/internal/hosted-delivery-protocol.d.ts +20 -0
- package/dist/internal/hosted-delivery-protocol.d.ts.map +1 -1
- package/dist/internal/hosted-delivery-protocol.js +51 -1
- package/dist/internal/hosted-delivery.d.ts.map +1 -1
- package/dist/internal/hosted-delivery.js +23 -25
- package/dist/internal/hosted-execution-diag.d.ts +12 -4
- package/dist/internal/hosted-execution-diag.d.ts.map +1 -1
- package/dist/internal/hosted-execution-diag.js +26 -4
- package/dist/internal/hosted-execution-flush.d.ts +1 -0
- package/dist/internal/hosted-execution-flush.d.ts.map +1 -1
- package/dist/internal/hosted-execution-flush.js +4 -2
- package/dist/internal/init-project.d.ts.map +1 -1
- package/dist/internal/init-project.js +4 -0
- package/dist/internal/server.d.ts.map +1 -1
- package/dist/internal/server.js +111 -48
- package/dist/internal/session-engine.d.ts +4 -1
- package/dist/internal/session-engine.d.ts.map +1 -1
- package/dist/internal/session-engine.js +39 -9
- package/dist/internal/skill-catalog.d.ts +28 -0
- package/dist/internal/skill-catalog.d.ts.map +1 -0
- package/dist/internal/skill-catalog.js +44 -0
- package/dist/playground/assets/index-DSMAewbx.css +1 -0
- package/dist/playground/assets/index-De_lpFxE.js +67 -0
- package/dist/playground/index.html +2 -2
- package/dist/types.d.ts +32 -3
- package/dist/types.d.ts.map +1 -1
- package/docs/README.md +2 -0
- package/docs/building-with-agents.md +96 -84
- package/docs/deployment.md +118 -364
- package/docs/evals.md +149 -673
- package/docs/guides/agent-to-agent.md +130 -48
- package/docs/guides/bitbucket.md +250 -49
- package/docs/guides/cloud-agents.md +119 -67
- package/docs/guides/convert-automation.md +157 -148
- package/docs/guides/github.md +261 -246
- package/docs/guides/gitlab.md +250 -51
- package/docs/guides/grokbot-agents.md +100 -55
- package/docs/guides/improve.md +112 -59
- package/docs/guides/jev.md +353 -0
- package/docs/guides/mcp-oauth.md +112 -122
- package/docs/guides/opentelemetry.md +97 -166
- package/docs/guides/slack.md +158 -228
- package/docs/guides/webhooks.md +155 -386
- package/docs/hillclimbing.md +102 -56
- package/docs/reference/cli.md +2 -0
- package/docs/reference/connections.md +1 -1
- package/docs/reference/evals.md +298 -0
- package/docs/reference/extensions.md +132 -203
- package/docs/reference/hooks.md +34 -34
- package/docs/reference/subagents.md +1 -1
- package/docs/reference/tools.md +4 -0
- package/docs/templates/thermo-quality-review.md +80 -0
- package/docs/templates/thermo-review.md +79 -0
- package/docs/troubleshooting.md +98 -67
- package/package.json +8 -1
- package/skills/github/SKILL.md +21 -13
- package/src/bin/agent-serve.ts +24 -3
- package/src/channels/checks.ts +8 -0
- package/src/channels/github/github-channel.ts +29 -19
- package/src/channels/origin/checks.ts +3 -1
- package/src/channels/slack/agentic-delivery.ts +2 -0
- package/src/channels/slack/dispatch.ts +131 -4
- package/src/channels/slack/placeholder.ts +51 -0
- package/src/channels/slack/slack-channel.ts +9 -2
- package/src/extensions/cursor-cloud-agents/skills/handoff.md +3 -7
- package/src/extensions/improve/extension.ts +4 -2
- package/src/extensions/improve/skills/yourself.ts +1 -1
- package/src/extensions/jev/extension.ts +95 -0
- package/src/extensions/jev/lib/evaluate.ts +289 -0
- package/src/extensions/jev/skills/gated-write.md +25 -0
- package/src/extensions/jev/skills/questions.md +33 -0
- package/src/extensions/jev/tools/evaluate.ts +90 -0
- package/src/extensions.ts +2 -0
- package/src/filesystem.ts +168 -65
- package/src/index.ts +3 -0
- package/src/internal/advertise-tools.ts +6 -0
- package/src/internal/cli-ax.ts +88 -15
- package/src/internal/continuation-identity.ts +1 -0
- package/src/internal/cursor-agent-template.ts +2 -0
- package/src/internal/discovery/connections.ts +21 -0
- package/src/internal/discovery/extensions.ts +12 -4
- package/src/internal/discovery/info.ts +1 -0
- package/src/internal/filesystem/tools.ts +2 -0
- package/src/internal/filesystem/walk.ts +60 -15
- package/src/internal/hosted-admission-adapter.ts +3 -0
- package/src/internal/hosted-delivery-protocol.ts +82 -1
- package/src/internal/hosted-delivery.ts +23 -0
- package/src/internal/hosted-execution-diag.ts +33 -4
- package/src/internal/hosted-execution-flush.ts +4 -0
- package/src/internal/init-project.ts +4 -0
- package/src/internal/server.ts +130 -53
- package/src/internal/session-engine.ts +46 -8
- package/src/internal/skill-catalog.ts +69 -0
- package/src/types.ts +33 -3
- package/templates/thermo-quality-review/README.md +35 -0
- package/templates/thermo-quality-review/agent/agent.ts +8 -0
- package/templates/thermo-quality-review/agent/channels/github.ts +42 -0
- package/templates/thermo-quality-review/agent/instructions.md +43 -0
- package/templates/thermo-quality-review/agent/tools/post_findings.ts +70 -0
- package/templates/thermo-quality-review/evals/evals.config.ts +5 -0
- package/templates/thermo-quality-review/evals/review.eval.ts +62 -0
- package/templates/thermo-quality-review/package.json +18 -0
- package/templates/thermo-quality-review/tsconfig.json +12 -0
- package/templates/thermo-review/README.md +35 -0
- package/templates/thermo-review/agent/agent.ts +8 -0
- package/templates/thermo-review/agent/channels/github.ts +42 -0
- package/templates/thermo-review/agent/instructions.md +40 -0
- package/templates/thermo-review/agent/tools/post_findings.ts +70 -0
- package/templates/thermo-review/evals/evals.config.ts +5 -0
- package/templates/thermo-review/evals/review.eval.ts +53 -0
- package/templates/thermo-review/package.json +18 -0
- package/templates/thermo-review/tsconfig.json +12 -0
- package/dist/docs/assets/building-with-agents.md.CEGVXkmO.js +0 -13
- package/dist/docs/assets/building-with-agents.md.CEGVXkmO.lean.js +0 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.DtXk1hy-.js +0 -1
- package/dist/docs/assets/chunks/channel.Bkv1N-gK.js +0 -1
- package/dist/docs/assets/chunks/classDiagram-6PBFFD2Q.CZDco1o8.js +0 -1
- package/dist/docs/assets/chunks/classDiagram-v2-HSJHXN6E.CZDco1o8.js +0 -1
- package/dist/docs/assets/chunks/clone.YSt_40_s.js +0 -1
- package/dist/docs/assets/chunks/framework.dypDpWZ3.js +0 -19
- package/dist/docs/assets/chunks/stateDiagram-v2-QKLJ7IA2.C4CLm481.js +0 -1
- package/dist/docs/assets/deployment.md.Dm4Qo3hp.js +0 -51
- package/dist/docs/assets/deployment.md.Dm4Qo3hp.lean.js +0 -1
- package/dist/docs/assets/evals.md.BLDRt5LH.js +0 -211
- package/dist/docs/assets/evals.md.BLDRt5LH.lean.js +0 -1
- package/dist/docs/assets/guides_agent-to-agent.md.BI0xclmy.js +0 -14
- package/dist/docs/assets/guides_agent-to-agent.md.BI0xclmy.lean.js +0 -1
- package/dist/docs/assets/guides_bitbucket.md.CTpCl__f.js +0 -10
- package/dist/docs/assets/guides_bitbucket.md.CTpCl__f.lean.js +0 -1
- package/dist/docs/assets/guides_cloud-agents.md.lSE_l7lH.js +0 -9
- package/dist/docs/assets/guides_cloud-agents.md.lSE_l7lH.lean.js +0 -1
- package/dist/docs/assets/guides_convert-automation.md.Ck6Cr68A.js +0 -12
- package/dist/docs/assets/guides_convert-automation.md.Ck6Cr68A.lean.js +0 -1
- package/dist/docs/assets/guides_github.md.D6ER29dG.js +0 -64
- package/dist/docs/assets/guides_github.md.D6ER29dG.lean.js +0 -1
- package/dist/docs/assets/guides_gitlab.md.P-TjBnS5.js +0 -14
- package/dist/docs/assets/guides_gitlab.md.P-TjBnS5.lean.js +0 -1
- package/dist/docs/assets/guides_grokbot-agents.md.WBZIOvkz.js +0 -9
- package/dist/docs/assets/guides_grokbot-agents.md.WBZIOvkz.lean.js +0 -1
- package/dist/docs/assets/guides_improve.md.BKaDuKKK.js +0 -14
- package/dist/docs/assets/guides_improve.md.BKaDuKKK.lean.js +0 -1
- package/dist/docs/assets/guides_mcp-oauth.md.DMNMpXtO.js +0 -28
- package/dist/docs/assets/guides_mcp-oauth.md.DMNMpXtO.lean.js +0 -1
- package/dist/docs/assets/guides_opentelemetry.md._CRfDyzH.js +0 -26
- package/dist/docs/assets/guides_opentelemetry.md._CRfDyzH.lean.js +0 -1
- package/dist/docs/assets/guides_slack.md.DdT8rmsj.js +0 -46
- package/dist/docs/assets/guides_slack.md.DdT8rmsj.lean.js +0 -1
- package/dist/docs/assets/guides_webhooks.md.aQW10HRe.js +0 -225
- package/dist/docs/assets/guides_webhooks.md.aQW10HRe.lean.js +0 -1
- package/dist/docs/assets/hillclimbing.md.Dq4kkVIL.js +0 -1
- package/dist/docs/assets/hillclimbing.md.Dq4kkVIL.lean.js +0 -1
- package/dist/docs/assets/reference_extensions.md.wlFD3cUR.js +0 -62
- package/dist/docs/assets/reference_extensions.md.wlFD3cUR.lean.js +0 -1
- package/dist/docs/assets/troubleshooting.md.BcgNoYtJ.js +0 -1
- package/dist/docs/assets/troubleshooting.md.BcgNoYtJ.lean.js +0 -1
- package/dist/playground/assets/index-BLlKgZtI.css +0 -1
- package/dist/playground/assets/index-CDS5p9sR.js +0 -67
package/dist/docs/evals.html
CHANGED
|
@@ -4,267 +4,128 @@
|
|
|
4
4
|
<meta charset="utf-8">
|
|
5
5
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
6
6
|
<title>Evals | Agent SDK</title>
|
|
7
|
-
<meta name="description" content="
|
|
7
|
+
<meta name="description" content="Protect agent decisions with fixed inputs, trajectory assertions, and regression checks that run locally or in CI.">
|
|
8
8
|
<meta name="generator" content="VitePress v1.6.4">
|
|
9
9
|
<link rel="preload stylesheet" href="/docs/assets/style.Cfo2vbik.css" as="style">
|
|
10
10
|
<link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
|
|
11
11
|
|
|
12
|
-
<script type="module" src="/docs/assets/app.
|
|
12
|
+
<script type="module" src="/docs/assets/app.CAeK13eM.js"></script>
|
|
13
13
|
<link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
|
|
14
|
-
<link rel="modulepreload" href="/docs/assets/chunks/framework.
|
|
15
|
-
<link rel="modulepreload" href="/docs/assets/chunks/theme.
|
|
14
|
+
<link rel="modulepreload" href="/docs/assets/chunks/framework.BNw1pucY.js">
|
|
15
|
+
<link rel="modulepreload" href="/docs/assets/chunks/theme.CXJ7PNwy.js">
|
|
16
16
|
<link rel="modulepreload" href="/docs/assets/chunks/katex.DkKDou_j.js">
|
|
17
|
-
<link rel="modulepreload" href="/docs/assets/chunks/dagre-KV5264BT.
|
|
18
|
-
<link rel="modulepreload" href="/docs/assets/chunks/cose-bilkent-S5V4N54A.
|
|
19
|
-
<link rel="modulepreload" href="/docs/assets/chunks/c4Diagram-AHTNJAMY.
|
|
20
|
-
<link rel="modulepreload" href="/docs/assets/chunks/flowDiagram-DWJPFMVM.
|
|
21
|
-
<link rel="modulepreload" href="/docs/assets/chunks/erDiagram-SMLLAGMA
|
|
22
|
-
<link rel="modulepreload" href="/docs/assets/chunks/gitGraphDiagram-UUTBAWPF.
|
|
23
|
-
<link rel="modulepreload" href="/docs/assets/chunks/ganttDiagram-T4ZO3ILL.
|
|
24
|
-
<link rel="modulepreload" href="/docs/assets/chunks/infoDiagram-42DDH7IO.
|
|
25
|
-
<link rel="modulepreload" href="/docs/assets/chunks/pieDiagram-DEJITSTG.
|
|
26
|
-
<link rel="modulepreload" href="/docs/assets/chunks/quadrantDiagram-34T5L4WZ.
|
|
27
|
-
<link rel="modulepreload" href="/docs/assets/chunks/xychartDiagram-5P7HB3ND.
|
|
28
|
-
<link rel="modulepreload" href="/docs/assets/chunks/requirementDiagram-MS252O5E.
|
|
29
|
-
<link rel="modulepreload" href="/docs/assets/chunks/sequenceDiagram-FGHM5R23.
|
|
30
|
-
<link rel="modulepreload" href="/docs/assets/chunks/classDiagram-6PBFFD2Q.
|
|
31
|
-
<link rel="modulepreload" href="/docs/assets/chunks/classDiagram-v2-HSJHXN6E.
|
|
32
|
-
<link rel="modulepreload" href="/docs/assets/chunks/stateDiagram-FHFEXIEX.
|
|
33
|
-
<link rel="modulepreload" href="/docs/assets/chunks/stateDiagram-v2-QKLJ7IA2.
|
|
34
|
-
<link rel="modulepreload" href="/docs/assets/chunks/journeyDiagram-VCZTEJTY.
|
|
35
|
-
<link rel="modulepreload" href="/docs/assets/chunks/timeline-definition-GMOUNBTQ.
|
|
36
|
-
<link rel="modulepreload" href="/docs/assets/chunks/mindmap-definition-QFDTVHPH.
|
|
37
|
-
<link rel="modulepreload" href="/docs/assets/chunks/kanban-definition-6JOO6SKY.
|
|
38
|
-
<link rel="modulepreload" href="/docs/assets/chunks/sankeyDiagram-XADWPNL6
|
|
39
|
-
<link rel="modulepreload" href="/docs/assets/chunks/diagram-TYMM5635.
|
|
40
|
-
<link rel="modulepreload" href="/docs/assets/chunks/diagram-MMDJMWI5.
|
|
41
|
-
<link rel="modulepreload" href="/docs/assets/chunks/blockDiagram-DXYQGD6D.
|
|
42
|
-
<link rel="modulepreload" href="/docs/assets/chunks/diagram-5BDNPKRD.
|
|
43
|
-
<link rel="modulepreload" href="/docs/assets/chunks/architectureDiagram-Q4EWVU46.
|
|
44
|
-
<link rel="modulepreload" href="/docs/assets/chunks/ishikawaDiagram-UXIWVN3A.
|
|
45
|
-
<link rel="modulepreload" href="/docs/assets/chunks/vennDiagram-DHZGUBPP.
|
|
46
|
-
<link rel="modulepreload" href="/docs/assets/chunks/diagram-G4DWMVQ6.
|
|
47
|
-
<link rel="modulepreload" href="/docs/assets/chunks/wardleyDiagram-NUSXRM2D.
|
|
17
|
+
<link rel="modulepreload" href="/docs/assets/chunks/dagre-KV5264BT.C9PZQK-S.js">
|
|
18
|
+
<link rel="modulepreload" href="/docs/assets/chunks/cose-bilkent-S5V4N54A.DVEa6fZp.js">
|
|
19
|
+
<link rel="modulepreload" href="/docs/assets/chunks/c4Diagram-AHTNJAMY.BRV8GPJJ.js">
|
|
20
|
+
<link rel="modulepreload" href="/docs/assets/chunks/flowDiagram-DWJPFMVM.ByWJXeYK.js">
|
|
21
|
+
<link rel="modulepreload" href="/docs/assets/chunks/erDiagram-SMLLAGMA.U2pR_OA7.js">
|
|
22
|
+
<link rel="modulepreload" href="/docs/assets/chunks/gitGraphDiagram-UUTBAWPF.Bpn01P7X.js">
|
|
23
|
+
<link rel="modulepreload" href="/docs/assets/chunks/ganttDiagram-T4ZO3ILL.OquF0Rtg.js">
|
|
24
|
+
<link rel="modulepreload" href="/docs/assets/chunks/infoDiagram-42DDH7IO.CqhknMWi.js">
|
|
25
|
+
<link rel="modulepreload" href="/docs/assets/chunks/pieDiagram-DEJITSTG.BoIDyuKF.js">
|
|
26
|
+
<link rel="modulepreload" href="/docs/assets/chunks/quadrantDiagram-34T5L4WZ.DLkpDytR.js">
|
|
27
|
+
<link rel="modulepreload" href="/docs/assets/chunks/xychartDiagram-5P7HB3ND.TRPe92m3.js">
|
|
28
|
+
<link rel="modulepreload" href="/docs/assets/chunks/requirementDiagram-MS252O5E.DqTVqSu2.js">
|
|
29
|
+
<link rel="modulepreload" href="/docs/assets/chunks/sequenceDiagram-FGHM5R23.BIp9602K.js">
|
|
30
|
+
<link rel="modulepreload" href="/docs/assets/chunks/classDiagram-6PBFFD2Q.Degh8l90.js">
|
|
31
|
+
<link rel="modulepreload" href="/docs/assets/chunks/classDiagram-v2-HSJHXN6E.Degh8l90.js">
|
|
32
|
+
<link rel="modulepreload" href="/docs/assets/chunks/stateDiagram-FHFEXIEX.COSXsD9I.js">
|
|
33
|
+
<link rel="modulepreload" href="/docs/assets/chunks/stateDiagram-v2-QKLJ7IA2.qrxrbFsX.js">
|
|
34
|
+
<link rel="modulepreload" href="/docs/assets/chunks/journeyDiagram-VCZTEJTY.Cg5f7oB3.js">
|
|
35
|
+
<link rel="modulepreload" href="/docs/assets/chunks/timeline-definition-GMOUNBTQ.CXdVqkLq.js">
|
|
36
|
+
<link rel="modulepreload" href="/docs/assets/chunks/mindmap-definition-QFDTVHPH.BLiysLpe.js">
|
|
37
|
+
<link rel="modulepreload" href="/docs/assets/chunks/kanban-definition-6JOO6SKY.Cx9YTwlU.js">
|
|
38
|
+
<link rel="modulepreload" href="/docs/assets/chunks/sankeyDiagram-XADWPNL6.CG_6FF7j.js">
|
|
39
|
+
<link rel="modulepreload" href="/docs/assets/chunks/diagram-TYMM5635.DW326M4K.js">
|
|
40
|
+
<link rel="modulepreload" href="/docs/assets/chunks/diagram-MMDJMWI5.BinJ5kWb.js">
|
|
41
|
+
<link rel="modulepreload" href="/docs/assets/chunks/blockDiagram-DXYQGD6D.JYq6w91N.js">
|
|
42
|
+
<link rel="modulepreload" href="/docs/assets/chunks/diagram-5BDNPKRD.DoN0uv3Y.js">
|
|
43
|
+
<link rel="modulepreload" href="/docs/assets/chunks/architectureDiagram-Q4EWVU46.CCXB8Uj5.js">
|
|
44
|
+
<link rel="modulepreload" href="/docs/assets/chunks/ishikawaDiagram-UXIWVN3A.C6xpR2af.js">
|
|
45
|
+
<link rel="modulepreload" href="/docs/assets/chunks/vennDiagram-DHZGUBPP.CZxGuc4r.js">
|
|
46
|
+
<link rel="modulepreload" href="/docs/assets/chunks/diagram-G4DWMVQ6.Czv3duqx.js">
|
|
47
|
+
<link rel="modulepreload" href="/docs/assets/chunks/wardleyDiagram-NUSXRM2D.6_irCgGJ.js">
|
|
48
48
|
<link rel="modulepreload" href="/docs/assets/chunks/virtual_mermaid-config.DDnGl6nM.js">
|
|
49
|
-
<link rel="modulepreload" href="/docs/assets/evals.md.
|
|
49
|
+
<link rel="modulepreload" href="/docs/assets/evals.md.D3Y3Aixt.lean.js">
|
|
50
50
|
<script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
|
|
51
51
|
<script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
|
|
52
52
|
<link rel="alternate" type="text/plain" href="/docs/llms.txt">
|
|
53
53
|
<link rel="alternate" type="text/markdown" href="/docs/evals.md">
|
|
54
54
|
</head>
|
|
55
55
|
<body>
|
|
56
|
-
<div id="app"><div class="Layout" data-v-76086e96><!--[--><!--]--><!--[--><span tabindex="-1" data-v-97fb5898></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-97fb5898>Skip to content</a><!--]--><!----><header class="VPNav" data-v-76086e96 data-v-781b1279><div class="VPNavBar" data-v-781b1279 data-v-b33015df><div class="wrapper" data-v-b33015df><div class="container" data-v-b33015df><div class="title" data-v-b33015df><div class="VPNavBarTitle has-sidebar" data-v-b33015df data-v-a0c466dc><a class="title" href="/docs/" data-v-a0c466dc><!--[--><!--]--><!----><span data-v-a0c466dc>Agent SDK</span><!--[--><!--[--><!--[--><!--[--><span class="agent-sdk-version" title="@cursor/july 0.1.111" data-v-97e8f22b>0.1.111</span><!--]--><!--]--><!--]--><!--]--></a></div></div><div class="content" data-v-b33015df><div class="content-body" data-v-b33015df><!--[--><!--]--><div class="VPNavBarSearch search" data-v-b33015df><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-b33015df data-v-8906c751><span id="main-nav-aria-label" class="visually-hidden" data-v-8906c751> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/quickstart.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Quickstart</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/guides/mcp-oauth.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Guides</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/guides/github.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Integrations</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/templates/security-reviewer.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Templates</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/reference/project-layout.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-b33015df data-v-959ed110><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-959ed110 data-v-79ad6d7f data-v-0a895de9><span class="check" data-v-0a895de9><span class="icon" data-v-0a895de9><!--[--><span class="vpi-sun sun" data-v-79ad6d7f></span><span class="vpi-moon moon" data-v-79ad6d7f></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-b33015df data-v-a713f501 data-v-1d7aa189><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-1d7aa189><span class="vpi-more-horizontal icon" data-v-1d7aa189></span></button><div class="menu" data-v-1d7aa189><div class="VPMenu" data-v-1d7aa189 data-v-e34b7e14><!----><!--[--><!--[--><!----><div class="group" data-v-a713f501><div class="item appearance" data-v-a713f501><p class="label" data-v-a713f501>Appearance</p><div class="appearance-action" data-v-a713f501><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-a713f501 data-v-79ad6d7f data-v-0a895de9><span class="check" data-v-0a895de9><span class="icon" data-v-0a895de9><!--[--><span class="vpi-sun sun" data-v-79ad6d7f></span><span class="vpi-moon moon" data-v-79ad6d7f></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-b33015df data-v-cb5d0edb><span class="container" data-v-cb5d0edb><span class="top" data-v-cb5d0edb></span><span class="middle" data-v-cb5d0edb></span><span class="bottom" data-v-cb5d0edb></span></span></button></div></div></div></div><div class="divider" data-v-b33015df><div class="divider-line" data-v-b33015df></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-76086e96 data-v-6f794310><div class="container" data-v-6f794310><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-6f794310><span class="vpi-align-left menu-icon" data-v-6f794310></span><span class="menu-text" data-v-6f794310>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-6f794310 data-v-bab84d9c><button data-v-bab84d9c>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-76086e96 data-v-4daa293d><div class="curtain" data-v-4daa293d></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-4daa293d><span class="visually-hidden" id="sidebar-aria-label" data-v-4daa293d> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Getting started</h2><!----></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/quickstart.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Building your first agent</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Integrations</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/github.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>GitHub</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/slack.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Slack</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/webhooks.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Webhooks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/convert-automation.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Cursor Automations</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/opentelemetry.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>OpenTelemetry</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/gitlab.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>GitLab</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/bitbucket.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Bitbucket</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Extensions</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/extensions.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/cloud-agents.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Cloud Agents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/grokbot-agents.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Grok Bot Agents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/improve.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Self-improving Agents</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible has-active" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Guides</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/mcp-oauth.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>MCP</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/agent-to-agent.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Agent-to-agent</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/deployment.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Deployment</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/evals.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Evals</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/hillclimbing.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Hillclimbing</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/building-with-agents.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Self Improving Agents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/troubleshooting.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Troubleshooting</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Templates</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/security-reviewer.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Security reviewer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/agentic-owners.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Agentic Approver</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/pr-autofixer.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>PR autofixer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/triage.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Triage agent</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Reference</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/project-layout.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Project layout</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/agent-config.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Agent config</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/instructions.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Instructions</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/tools.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Tools</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/prompt.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>prompt</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/skills.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Skills</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/connections.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>MCP Connections</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/subagents.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Subagents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/channels.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Channels</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/schedules.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Schedules & reminders</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/hooks.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Hooks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/artifacts.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Artifacts</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/sessions.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Sessions & streaming</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/playground.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Playground</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/cli.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>CLI</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/http-api.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>HTTP API</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-76086e96 data-v-258cbba4><div class="VPDoc has-sidebar has-aside" data-v-258cbba4 data-v-afe580da><!--[--><!--]--><div class="container" data-v-afe580da><div class="aside" data-v-afe580da><div class="aside-curtain" data-v-afe580da></div><div class="aside-container" data-v-afe580da><div class="aside-content" data-v-afe580da><div class="VPDocAside" data-v-afe580da data-v-fb5275a4><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-fb5275a4 data-v-b3783f18><div class="content" data-v-b3783f18><div class="outline-marker" data-v-b3783f18></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-b3783f18>On this page</div><ul class="VPDocOutlineItem root" data-v-b3783f18 data-v-72702c6e><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-fb5275a4></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-afe580da><div class="content-container" data-v-afe580da><!--[--><!--]--><main class="main" data-v-afe580da><div style="position:relative;" class="vp-doc _docs_evals" data-v-afe580da><div><h1 id="evals" tabindex="-1">Evals <a class="header-anchor" href="#evals" aria-label="Permalink to "Evals""></a></h1><p>An eval sends a fixed message to your agent and asserts over the trajectory it records: the turn completed, the right tool ran with the right input, the reply has the right shape. Evals are how you know a prompt tweak helped, a refactor didn't regress the agent, and last month's fix still holds.</p><p>Nothing is mocked. The runner starts (or targets) a real agent server, drives sessions over the public API, and grades the events it gets back. The model runs and server tools execute, so <a href="#keep-side-effects-out-of-eval-sessions">keep side effects out of eval sessions</a> before you point an eval at an agent that posts anywhere.</p><h2 id="evals-hooks-or-hillclimbing" tabindex="-1">Evals, hooks, or hillclimbing? <a class="header-anchor" href="#evals-hooks-or-hillclimbing" aria-label="Permalink to "Evals, hooks, or hillclimbing?""></a></h2><p>All three read the same session event stream. Pick by the question you are asking.</p><table tabindex="0"><thead><tr><th>You want to</th><th>Use</th></tr></thead><tbody><tr><td>Gate one fixed input's behavior, locally and in CI</td><td>Evals (this page)</td></tr><tr><td>Observe every live session: metrics, audit, alerts</td><td><a href="./reference/hooks.html">Hooks</a></td></tr><tr><td>Improve an agent one measured round at a time</td><td><a href="./hillclimbing.html">Hillclimbing</a>; each kept win lands an eval</td></tr></tbody></table><p><a href="./reference/hooks.html#hooks-channel-events-or-evals">Hooks, channel events, or evals?</a> has the side-by-side table.</p><h3 id="when-not-to-write-an-eval" tabindex="-1">When not to write an eval <a class="header-anchor" href="#when-not-to-write-an-eval" aria-label="Permalink to "When not to write an eval""></a></h3><ul><li>Test a server tool's own logic with <code>agent-sdk call <tool> --dir . --input '{...}'</code> or a unit test. No model turn, no credential.</li><li>Explore a prompt with <code>agent-sdk run --dir . --message "..."</code> and read the trajectory. Write the eval once you know which decision to gate.</li><li>Stop a bad turn while it runs with <a href="./reference/tools.html#gate-a-tool-on-human-approval"><code>needsApproval</code></a> on the tool or <code>defineResult</code>. Evals grade after the fact.</li></ul><h2 id="write-your-first-eval" tabindex="-1">Write your first eval <a class="header-anchor" href="#write-your-first-eval" aria-label="Permalink to "Write your first eval""></a></h2><p>Evals live under the project-root <code>evals/</code> directory, a sibling of <code>agent/</code>. <code>agent/evals/</code> is silently ignored. Discovery loads every <code>.eval.ts</code> (or <code>.eval.js</code>) file under <code>evals/</code>, plus one config file.</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>my-agent/</span></span>
|
|
57
|
-
<span class="line"><span> agent/</span></span>
|
|
58
|
-
<span class="line"><span> agent.ts</span></span>
|
|
59
|
-
<span class="line"><span> tools/inspect_pr.ts</span></span>
|
|
60
|
-
<span class="line"><span> evals/</span></span>
|
|
61
|
-
<span class="line"><span> evals.config.ts # required to run: maxConcurrency</span></span>
|
|
62
|
-
<span class="line"><span> readiness.eval.ts # id: readiness</span></span>
|
|
63
|
-
<span class="line"><span> prs.eval.ts # cases: prs/checkout, prs/search</span></span></code></pre></div><p>An eval is a single <code>async test(t)</code>. You drive the agent with <code>t.send</code> and assert on the recorded run with the same <code>t</code>:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/readiness.eval.ts</span></span>
|
|
56
|
+
<div id="app"><div class="Layout" data-v-76086e96><!--[--><!--]--><!--[--><span tabindex="-1" data-v-97fb5898></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-97fb5898>Skip to content</a><!--]--><!----><header class="VPNav" data-v-76086e96 data-v-781b1279><div class="VPNavBar" data-v-781b1279 data-v-b33015df><div class="wrapper" data-v-b33015df><div class="container" data-v-b33015df><div class="title" data-v-b33015df><div class="VPNavBarTitle has-sidebar" data-v-b33015df data-v-a0c466dc><a class="title" href="/docs/" data-v-a0c466dc><!--[--><!--]--><!----><span data-v-a0c466dc>Agent SDK</span><!--[--><!--[--><!--[--><!--[--><span class="agent-sdk-version" title="@cursor/july 0.1.113" data-v-97e8f22b>0.1.113</span><!--]--><!--]--><!--]--><!--]--></a></div></div><div class="content" data-v-b33015df><div class="content-body" data-v-b33015df><!--[--><!--]--><div class="VPNavBarSearch search" data-v-b33015df><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-b33015df data-v-8906c751><span id="main-nav-aria-label" class="visually-hidden" data-v-8906c751> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/quickstart.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Quickstart</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/guides/mcp-oauth.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Guides</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/guides/github.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Integrations</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/templates/security-reviewer.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Templates</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/reference/project-layout.html" tabindex="0" data-v-8906c751 data-v-e9bcc08e><!--[--><span data-v-e9bcc08e>Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-b33015df data-v-959ed110><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-959ed110 data-v-79ad6d7f data-v-0a895de9><span class="check" data-v-0a895de9><span class="icon" data-v-0a895de9><!--[--><span class="vpi-sun sun" data-v-79ad6d7f></span><span class="vpi-moon moon" data-v-79ad6d7f></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-b33015df data-v-a713f501 data-v-1d7aa189><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-1d7aa189><span class="vpi-more-horizontal icon" data-v-1d7aa189></span></button><div class="menu" data-v-1d7aa189><div class="VPMenu" data-v-1d7aa189 data-v-e34b7e14><!----><!--[--><!--[--><!----><div class="group" data-v-a713f501><div class="item appearance" data-v-a713f501><p class="label" data-v-a713f501>Appearance</p><div class="appearance-action" data-v-a713f501><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-a713f501 data-v-79ad6d7f data-v-0a895de9><span class="check" data-v-0a895de9><span class="icon" data-v-0a895de9><!--[--><span class="vpi-sun sun" data-v-79ad6d7f></span><span class="vpi-moon moon" data-v-79ad6d7f></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-b33015df data-v-cb5d0edb><span class="container" data-v-cb5d0edb><span class="top" data-v-cb5d0edb></span><span class="middle" data-v-cb5d0edb></span><span class="bottom" data-v-cb5d0edb></span></span></button></div></div></div></div><div class="divider" data-v-b33015df><div class="divider-line" data-v-b33015df></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-76086e96 data-v-6f794310><div class="container" data-v-6f794310><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-6f794310><span class="vpi-align-left menu-icon" data-v-6f794310></span><span class="menu-text" data-v-6f794310>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-6f794310 data-v-bab84d9c><button data-v-bab84d9c>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-76086e96 data-v-4daa293d><div class="curtain" data-v-4daa293d></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-4daa293d><span class="visually-hidden" id="sidebar-aria-label" data-v-4daa293d> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Getting started</h2><!----></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/quickstart.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Building your first agent</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Integrations</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/github.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>GitHub</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/slack.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Slack</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/webhooks.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Webhooks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/convert-automation.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Cursor Automations</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/opentelemetry.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>OpenTelemetry</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/gitlab.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>GitLab</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/bitbucket.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Bitbucket</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Extensions</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/extensions.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/cloud-agents.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Cloud agents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/grokbot-agents.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Grok Bot agents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/improve.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Self-improvement</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/jev.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Jev</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible has-active" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Guides</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/mcp-oauth.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>MCP</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/guides/agent-to-agent.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Peer agents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/deployment.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Deployment</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/evals.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Evals</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/hillclimbing.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Hillclimbing</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/building-with-agents.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Build with a coding agent</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/troubleshooting.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Troubleshooting</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Templates</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/security-reviewer.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Security reviewer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/thermo-review.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Thermo review</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/thermo-quality-review.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Thermo quality review</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/agentic-owners.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Agentic Approver</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/pr-autofixer.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>PR autofixer</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/templates/triage.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Triage agent</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-095feacd><section class="VPSidebarItem level-0 collapsible" data-v-095feacd data-v-d79cbf7e><div class="item" role="button" tabindex="0" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><h2 class="text" data-v-d79cbf7e>Reference</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-d79cbf7e><span class="vpi-chevron-right caret-icon" data-v-d79cbf7e></span></div></div><div class="items" data-v-d79cbf7e><!--[--><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/project-layout.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Project layout</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/agent-config.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Agent config</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/instructions.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Instructions</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/tools.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Tools</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/prompt.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>prompt</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/skills.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Skills</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/connections.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>MCP Connections</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/subagents.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Subagents</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/channels.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Channels</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/schedules.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Schedules & reminders</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/hooks.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Hooks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/evals.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Evals</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/artifacts.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Artifacts</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/sessions.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Sessions & streaming</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/playground.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>Playground</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/cli.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>CLI</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-d79cbf7e><div class="item" data-v-d79cbf7e><div class="indicator" data-v-d79cbf7e></div><a class="VPLink link link" href="/docs/reference/http-api.html" data-v-d79cbf7e><!--[--><p class="text" data-v-d79cbf7e>HTTP API</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-76086e96 data-v-258cbba4><div class="VPDoc has-sidebar has-aside" data-v-258cbba4 data-v-afe580da><!--[--><!--]--><div class="container" data-v-afe580da><div class="aside" data-v-afe580da><div class="aside-curtain" data-v-afe580da></div><div class="aside-container" data-v-afe580da><div class="aside-content" data-v-afe580da><div class="VPDocAside" data-v-afe580da data-v-fb5275a4><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-fb5275a4 data-v-b3783f18><div class="content" data-v-b3783f18><div class="outline-marker" data-v-b3783f18></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-b3783f18>On this page</div><ul class="VPDocOutlineItem root" data-v-b3783f18 data-v-72702c6e><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-fb5275a4></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-afe580da><div class="content-container" data-v-afe580da><!--[--><!--]--><main class="main" data-v-afe580da><div style="position:relative;" class="vp-doc _docs_evals" data-v-afe580da><div><h1 id="evals" tabindex="-1">Evals <a class="header-anchor" href="#evals" aria-label="Permalink to "Evals""></a></h1><p>An eval sends a fixed input to the real agent and checks the resulting trajectory: whether the turn succeeded, which tools ran, and what shape the answer took. Use evals to protect behavior you already understand, not to discover what the prompt should do.</p><p>Evals run real model turns and real tools. Guard external writes before running a suite against an agent that can post, merge, or deploy.</p><h2 id="protect-a-tool-decision" tabindex="-1">Protect a tool decision <a class="header-anchor" href="#protect-a-tool-decision" aria-label="Permalink to "Protect a tool decision""></a></h2><p>This smoke case asks for a PR verdict, requires the read-only inspection tool, and fails if the agent tries to approve. The CLI reports all three decisions together instead of stopping at the first miss.</p><p>Use the <a href="./guides/jev.html">Jev extension</a> when host code needs a typed choice, score, or boolean before it writes.</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/readiness.eval.ts</span></span>
|
|
64
57
|
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineEval, includes } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "@cursor/july/evals"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
|
|
65
58
|
<span class="line"></span>
|
|
66
59
|
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEval</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
|
|
67
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> description: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Inspects a PR without approving it."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
68
60
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> tags: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"smoke"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">],</span></span>
|
|
69
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> timeoutMs: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">120_000</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
70
61
|
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> async</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">t</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
|
|
71
62
|
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
|
|
72
63
|
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "Is https://github.com/acme/checkout/pull/42 ready to approve?"</span></span>
|
|
73
64
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> );</span></span>
|
|
65
|
+
<span class="line"></span>
|
|
74
66
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
|
|
75
67
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"inspect_pr"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
|
|
76
68
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">notCalledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"approve_pr"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
|
|
77
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(t.reply, </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">includes</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">ready</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">approve</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">i</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">));</span></span>
|
|
69
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(t.reply, </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">includes</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">ready</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">blocked</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">approve</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">i</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">));</span></span>
|
|
78
70
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
|
|
79
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/evals.config.ts</span></span>
|
|
71
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p>Evals live in the project-root <code>evals/</code> directory, beside <code>agent/</code>. Add the required concurrency config once:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/evals.config.ts</span></span>
|
|
80
72
|
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineEvalConfig } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "@cursor/july/evals"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
|
|
81
73
|
<span class="line"></span>
|
|
82
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEvalConfig</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({
|
|
83
|
-
<span class="line"><span style="--shiki-light:#
|
|
84
|
-
<span class="line"><span>
|
|
85
|
-
<span class="line"><span>
|
|
86
|
-
<span class="line"><span> ✓ notCalledTool(approve_pr)</span></span>
|
|
87
|
-
<span class="line"><span> ✓ check(includes)</span></span>
|
|
88
|
-
<span class="line"><span></span></span>
|
|
89
|
-
<span class="line"><span>1 passed, 0 failed, 1 total</span></span>
|
|
90
|
-
<span class="line"><span>artifacts: <project state directory>/evals/2026-09-11T15-02-11-402Z</span></span></code></pre></div><p>Every local run writes each case's assertions, inputs, tool calls, and <code>t.log</code> lines under that artifacts directory. Open <code>evals/<case-id>.json</code> there when a case fails; see <a href="#where-results-land">Where results land</a>.</p><h2 id="name-cases-by-path" tabindex="-1">Name cases by path <a class="header-anchor" href="#name-cases-by-path" aria-label="Permalink to "Name cases by path""></a></h2><p>The file path is the eval's identity, so you don't author an id. <code>evals/builds/api.eval.ts</code> becomes <code>builds/api</code>. An <code>index</code> filename collapses to its directory: <code>evals/builds/index.eval.ts</code> becomes <code>builds</code>.</p><p>One file can hold several datapoints through <code>cases</code>. Provide either <code>test</code> or <code>cases</code>, not both. Each case id becomes <code><fileId>/<case.id></code>:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/prs.eval.ts: prs/checkout, prs/search</span></span>
|
|
91
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEval</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
|
|
92
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> tags: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"smoke"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"prs"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">],</span></span>
|
|
93
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> cases: [</span></span>
|
|
94
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> {</span></span>
|
|
95
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> id: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"checkout"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
96
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> description: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Checkout PR readiness."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
97
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> async</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">t</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
|
|
98
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
|
|
99
|
-
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "Is https://github.com/acme/checkout/pull/42 ready to approve?"</span></span>
|
|
100
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> );</span></span>
|
|
101
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
|
|
102
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"inspect_pr"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
|
|
103
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
|
|
104
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
|
|
105
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> {</span></span>
|
|
106
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> id: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"search"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
107
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> async</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">t</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
|
|
108
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
|
|
109
|
-
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "Check https://github.com/acme/search/pull/7 before approval."</span></span>
|
|
110
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> );</span></span>
|
|
111
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
|
|
112
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"inspect_pr"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
|
|
113
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
|
|
114
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
|
|
115
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> ],</span></span>
|
|
116
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p>Case ids are single path segments, unique within the file. A case can set its own <code>description</code>, <code>tags</code>, <code>timeoutMs</code>, <code>iterations</code>, <code>judge</code>, <code>reporters</code>, and <code>metadata</code>. A case-level value replaces the file-level one for that datapoint, except <code>metadata</code>, which merges with case keys winning, and <code>reporters</code>, which adds to the file's list. <code>metadata</code> is free-form data carried onto the result and every reporter.</p><p>A file may instead export an array of <code>defineEval</code> calls to fan out over a dataset. Ids are then the file id plus a zero-padded index (<code>sql/0000</code>, <code>sql/0001</code>, ...); see <a href="#load-a-dataset">Load a dataset</a>. Prefer <code>cases</code> when datapoints are hand-written and deserve stable names.</p><h3 id="iterations" tabindex="-1">Iterations <a class="header-anchor" href="#iterations" aria-label="Permalink to "Iterations""></a></h3><p><code>iterations</code> (file or case, default <code>1</code>, cap <code>100</code>) runs a datapoint repeatedly. Discovery expands <code>iterations: 3</code> on case <code>nyc</code> to runnable ids <code>weather/nyc/1</code>, <code>weather/nyc/2</code>, and <code>weather/nyc/3</code>. The filter <code>weather/nyc</code> still selects all three. Each expanded case exposes <code>t.iteration</code> and <code>t.iterations</code>.</p><p><code>maxConcurrency</code> counts authored datapoints, not expanded iterations. Iterations of one datapoint share a concurrency slot and run in sequence, so a suite of 11 cases with 3 iterations each and <code>maxConcurrency: 20</code> has at most 11 cases in flight.</p><h2 id="drive-the-agent-with-t-send" tabindex="-1">Drive the agent with <code>t.send</code> <a class="header-anchor" href="#drive-the-agent-with-t-send" aria-label="Permalink to "Drive the agent with `t.send`""></a></h2><p><code>t.send(message, options?)</code> runs one turn and waits for it to settle: complete, park on an approval request, or fail. Several sends in one case share the session, which is how you write multi-turn evals.</p><p>Each send resolves to a turn result: <code>message</code> (the assistant text), <code>sessionId</code>, <code>events</code>, <code>toolCalls</code> (tool names in order), <code>ok</code>, and <code>index</code>. The turn carries the same assertion vocabulary as <code>t</code>, scoped to that turn, so you can grade an intermediate turn before the next send overwrites <code>t.reply</code>. <code>turn.expectOk()</code> throws when the turn failed, for later steps that depend on it.</p><p>Read the whole case with <code>t.reply</code> (last assistant text), <code>t.events</code> (every event so far), <code>t.turns</code> (settled turns, oldest first), and <code>t.sessionId</code>. <code>t.signal</code> aborts when the case hits its timeout; pass it to your own async work.</p><p>Three options apply on the first send only, because they shape session creation:</p><table tabindex="0"><thead><tr><th>Option</th><th>Effect</th></tr></thead><tbody><tr><td><code>workspaceFiles</code></td><td><code>{ path: contents }</code> seeded into the local session workspace. Prefer this over machine-local paths</td></tr><tr><td><code>workspaceDir</code></td><td>Absolute harness cwd for the local runtime</td></tr><tr><td><code>cloud</code></td><td>Per-session cloud options merged over the agent's static <code>cloud</code> config. Pin a fixture repo here for cloud evals instead of on the agent's default <code>cloud.repos</code>. On the cloud runtime, seeded files reach the agent as described under <a href="./reference/agent-config.html#choose-a-runtime">Choose a runtime</a></td></tr></tbody></table><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Review pr/diff.patch and post findings."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, {</span></span>
|
|
117
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> workspaceFiles: {</span></span>
|
|
118
|
-
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "pr/diff.patch"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
|
|
119
|
-
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "diff --git a/app/routes/search.ts b/app/routes/search.ts"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
120
|
-
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "+res.send(`<h1>Results for ${req.query.q}</h1>`);"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
121
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> ].</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">join</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">),</span></span>
|
|
122
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
|
|
123
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><h2 id="assert-over-the-trajectory" tabindex="-1">Assert over the trajectory <a class="header-anchor" href="#assert-over-the-trajectory" aria-label="Permalink to "Assert over the trajectory""></a></h2><p>Assertions record; they never throw. One run reports every failure instead of dying on the first. Assertions on <code>t</code> read the whole run. Assertions on a turn read only that turn.</p><table tabindex="0"><thead><tr><th>Gate</th><th>Checks</th></tr></thead><tbody><tr><td><code>t.succeeded()</code></td><td>the run did not fail and is not parked on an unanswered approval</td></tr><tr><td><code>t.parked()</code></td><td>the run cleanly parked on an unanswered approval request</td></tr><tr><td><code>t.messageIncludes(token)</code></td><td>the joined assistant text matches a string or <code>RegExp</code></td></tr><tr><td><code>t.calledTool(name, matcher?)</code></td><td>a matching call to <code>name</code> happened</td></tr><tr><td><code>t.notCalledTool(name)</code></td><td>no request for <code>name</code>, in any lifecycle state</td></tr><tr><td><code>t.loadedSkill(name)</code></td><td>the agent opened <code>skills/<name>/SKILL.md</code> (read, grep, or shell <code>cat</code>)</td></tr><tr><td><code>t.toolOrder(names)</code></td><td>tool requests appear in this relative order; extra calls allowed</td></tr><tr><td><code>t.usedNoTools()</code></td><td>no tool calls at all</td></tr><tr><td><code>t.maxToolCalls(max)</code></td><td>at most <code>max</code> tool calls</td></tr><tr><td><code>t.noFailedActions()</code></td><td>no tool call reported an error</td></tr><tr><td><code>t.calledSubagent(name, matcher?)</code></td><td>a matching subagent delegation happened</td></tr><tr><td><code>t.taggedArtifact(kind?, predicate?)</code></td><td>at least one <a href="./reference/artifacts.html">artifact</a> was tagged</td></tr><tr><td><code>t.event(type, matcher?)</code></td><td>at least one matching <a href="./reference/sessions.html#which-events-can-i-stream">event</a> of <code>type</code></td></tr><tr><td><code>t.notEvent(type, matcher?)</code></td><td>no matching event of <code>type</code></td></tr><tr><td><code>t.eventOrder(matchers)</code></td><td>matching event groups occur in this relative order</td></tr><tr><td><code>t.eventsSatisfy(label, predicate)</code></td><td>your predicate over the typed event stream</td></tr><tr><td><code>t.check(value, expectation)</code></td><td>any value, against a <a href="#grade-values-with-expectation-builders">builder</a></td></tr><tr><td><code>t.score(name, value)</code></td><td>a 0-1 score you computed; soft until you add a bar</td></tr></tbody></table><p>Three more assertions gate and return the matched fact. They stop the test body when nothing matches, without a duplicate execution error. <code>t.requireToolCall(name, matcher?)</code> returns the call so later code can read its <code>input</code> and <code>output</code>. <code>t.requireInputRequest(filter?)</code> returns the single pending approval request. <code>await t.require(value, expectation)</code> does the same for a value check.</p><p>A case with no assertions passes when at least one turn completed. Add <code>t.succeeded()</code> and behavior gates anyway. They make the contract visible in review.</p><h3 id="what-good-cases-assert" tabindex="-1">What good cases assert <a class="header-anchor" href="#what-good-cases-assert" aria-label="Permalink to "What good cases assert""></a></h3><p>Gate decisions and shape, not prose. Model wording varies run to run. Tool choice, tool avoidance, and output structure are the stable contract.</p><ol><li><code>t.succeeded()</code>: always, first.</li><li>The tool decision: <code>calledTool</code> for the intended path, <code>notCalledTool</code> for the likely wrong alternative. The pair is stronger than either alone.</li><li>Output shape: a regex for the contract (<code>/ready|blocked/i</code>, a JSON marker, a findings-block fence), never exact sentences.</li><li>For structured output, parse <code>t.reply</code> and check fields with <code>matches</code> or <code>satisfies</code> instead of substring-matching JSON.</li></ol><p>The common failure modes: asserting exact phrasing, packing more than about five gates into one case (split it), and cases that depend on live external state that drifts (pin the input).</p><h3 id="narrow-tool-assertions-with-matchers" tabindex="-1">Narrow tool assertions with matchers <a class="header-anchor" href="#narrow-tool-assertions-with-matchers" aria-label="Permalink to "Narrow tool assertions with matchers""></a></h3><p>With no matcher, <code>calledTool</code> is request-based: a requested call counts even before its result arrives. A matcher narrows it:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"inspect_pr"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, { status: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"completed"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> });</span></span>
|
|
124
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"apply_agents"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, { input: { verdict: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"update"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> } });</span></span>
|
|
125
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"bash"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, { input: { command:</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">^</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">gh pr view</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }, count: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> });</span></span>
|
|
126
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"read_file"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, {</span></span>
|
|
127
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> output</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: (</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">value</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=></span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> String</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(value).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">includes</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"TODO"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">),</span></span>
|
|
128
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p><code>input</code>, <code>output</code>, and <code>count</code> accept a literal, a <code>RegExp</code>, or a predicate. Object literals partial-deep-match, so <code>{ verdict: "update" }</code> matches arguments that also carry other keys. <code>status</code> is one of <code>completed</code>, <code>failed</code>, <code>pending</code>, or <code>rejected</code> (a human denied the approval). <code>calledSubagent</code> takes <code>{ output, status, count, callId }</code>. <code>event</code>, <code>notEvent</code>, and <code>eventOrder</code> take <code>{ data, count }</code>.</p><h3 id="grade-values-with-expectation-builders" tabindex="-1">Grade values with expectation builders <a class="header-anchor" href="#grade-values-with-expectation-builders" aria-label="Permalink to "Grade values with expectation builders""></a></h3><p><code>t.check(value, expectation)</code> grades any value: <code>t.reply</code>, a parsed JSON field, a tool's output.</p><table tabindex="0"><thead><tr><th>Builder</th><th>Checks</th><th>Severity</th></tr></thead><tbody><tr><td><code>includes(string | RegExp)</code></td><td>substring or match; structured values are stringified first</td><td>gate</td></tr><tr><td><code>equals(value)</code></td><td>deep equality</td><td>gate</td></tr><tr><td><code>matches(schema)</code></td><td>a Standard Schema (Zod, Valibot, ...) or anything with <code>safeParse</code></td><td>gate</td></tr><tr><td><code>similarity(expected)</code></td><td>normalized text similarity, 0-1</td><td>soft</td></tr><tr><td><code>satisfies(predicate, label)</code></td><td>your predicate; <code>label</code> is the failure detail</td><td>gate</td></tr></tbody></table><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { matches, satisfies } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "@cursor/july/evals"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
|
|
74
|
+
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEvalConfig</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
|
|
75
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> maxConcurrency: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">20</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
76
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span></span>
|
|
77
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> smoke</span></span></code></pre></div><h2 id="check-answer-shape-or-quality" tabindex="-1">Check answer shape or quality <a class="header-anchor" href="#check-answer-shape-or-quality" aria-label="Permalink to "Check answer shape or quality""></a></h2><p>Prefer a deterministic shape check when the output has a contract. The failure tells you which field was wrong, and the case remains stable when the model changes its wording.</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { matches } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "@cursor/july/evals"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
|
|
129
78
|
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { z } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "zod"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
|
|
130
79
|
<span class="line"></span>
|
|
131
80
|
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> verdict</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> JSON</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">parse</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(t.reply </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">??</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "{}"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
|
|
132
81
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
|
|
133
82
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> verdict,</span></span>
|
|
134
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> matches</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(
|
|
135
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
136
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
137
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
138
|
-
<span class="line"><span style="--shiki-light:#
|
|
139
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)
|
|
140
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
141
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
142
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
143
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
144
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.judge.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">closedQA</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"names the failing step"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, { on: summary.message }).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">gate</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><p>Judge assertions are soft by default, so a judge never fails a build until you give it a bar with <code>.atLeast()</code> or promote it with <code>.gate()</code>. The recorded detail names the choice the judge made and its rationale.</p><p>The judge model comes from <code>defineEvalConfig({ judge })</code>, <code>defineEval({ judge })</code>, a case-level <code>judge</code>, or a per-call <code>{ model }</code>. The nearest one wins. A judge call with no model configured fails the case. A judge that cannot reach a model (no credential) ends the case as <code>skipped</code>, unless a deterministic gate already failed.</p><p>For a domain-specific judge whose verdict is not a single score, <code>t.judge.model(prompt)</code> sends a raw prompt to the same model and returns the reply. Record the parsed result with <code>t.score</code> or <code>t.check</code>. Anything derived from the agent under test is untrusted input to your prompt: wrap it with <code>fenceUntrusted</code> and include <code>EVAL_JUDGE_INJECTION_GUARD</code>, as the built-in graders do.</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> {</span></span>
|
|
145
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> EVAL_JUDGE_INJECTION_GUARD,</span></span>
|
|
146
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> fenceUntrusted,</span></span>
|
|
147
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">} </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "@cursor/july/evals"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
|
|
148
|
-
<span class="line"></span>
|
|
149
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> gold</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"XSS in search.ts"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"open redirect in login.ts"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">];</span></span>
|
|
150
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> reply</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.judge.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">model</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
|
|
151
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [</span></span>
|
|
152
|
-
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "For each GOLD finding, answer whether SUBMISSION reports it."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
153
|
-
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "Reply with one line per finding: <index> YES|NO."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
154
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> EVAL_JUDGE_INJECTION_GUARD</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
155
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> fenceUntrusted</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"GOLD"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, gold.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">map</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">((</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">g</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">i</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=></span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> `${</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">i</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> +</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> 1</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">}. ${</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">g</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">}`</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">join</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)),</span></span>
|
|
156
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> fenceUntrusted</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"SUBMISSION"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, t.reply </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">??</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> ""</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">),</span></span>
|
|
157
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> ].</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">join</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\n\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span>
|
|
158
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
|
|
159
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> hits</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> reply.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">match</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">\b</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">YES</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">\b</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">g</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)?.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">length</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> ??</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> 0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
|
|
160
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">score</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"recall"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, hits </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">/</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> gold.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">length</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">atLeast</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.5</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><h2 id="keep-side-effects-out-of-eval-sessions" tabindex="-1">Keep side effects out of eval sessions <a class="header-anchor" href="#keep-side-effects-out-of-eval-sessions" aria-label="Permalink to "Keep side effects out of eval sessions""></a></h2><p>Eval sessions run the real agent, tools included. A reviewer that comments on GitHub or posts to Slack will do so from an eval unless the tool checks the session's purpose. Eval sessions carry <code>purpose: "eval"</code>; live traffic carries <code>"live"</code>. Branch on it in the tool, hook, or result handler that actuates:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// agent/tools/post_findings.ts</span></span>
|
|
161
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">async </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">execute</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({ findings }, ctx) {</span></span>
|
|
162
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> if</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> (ctx.session.purpose </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">===</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "eval"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
|
|
163
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { posted: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">false</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, reason: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"eval"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, count: findings.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">length</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> };</span></span>
|
|
164
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
|
|
165
|
-
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> // post the review</span></span>
|
|
166
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Return a shaped result instead of throwing, so the eval can still assert <code>t.calledTool("post_findings", { input: ... })</code> on the decision. The same check belongs in <a href="./reference/hooks.html">hooks</a> that meter or page and in <code>defineResult</code> commits.</p><h2 id="worked-examples" tabindex="-1">Worked examples <a class="header-anchor" href="#worked-examples" aria-label="Permalink to "Worked examples""></a></h2><h3 id="multi-turn-grade-each-turn" tabindex="-1">Multi-turn: grade each turn <a class="header-anchor" href="#multi-turn-grade-each-turn" aria-label="Permalink to "Multi-turn: grade each turn""></a></h3><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">// evals/intro.eval.ts</span></span>
|
|
167
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineEval, includes, satisfies } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "@cursor/july/evals"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
|
|
168
|
-
<span class="line"></span>
|
|
169
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEval</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
|
|
170
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> description: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Introduces itself once; a repeat mention gets a short ack."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
171
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> async</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">t</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
|
|
172
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> intro</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Meet Jenny! @Jenny introduce yourself."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
|
|
173
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> intro.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">expectOk</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
|
|
174
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(intro.message, </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">includes</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">jenny</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">i</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">));</span></span>
|
|
83
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> matches</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span></span>
|
|
84
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> z.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">object</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
|
|
85
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> ready: z.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">boolean</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(),</span></span>
|
|
86
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> blockers: z.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">array</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(z.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">string</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">()),</span></span>
|
|
87
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> })</span></span>
|
|
88
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> )</span></span>
|
|
89
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><p>Use a judge when correctness depends on meaning that a schema, regex, or known value cannot capture:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.judge</span></span>
|
|
90
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> .</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">factuality</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"The lint step failed on src/sidebar.ts."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span>
|
|
91
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> .</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">atLeast</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.8</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><p>Judge checks are tracked scores until you give them a hard gate. Configure the judge model in <code>evals.config.ts</code>. The <a href="./reference/evals.html#judges">Evals reference</a> owns grader options and severity rules.</p><h2 id="test-a-conversation-or-approval" tabindex="-1">Test a conversation or approval <a class="header-anchor" href="#test-a-conversation-or-approval" aria-label="Permalink to "Test a conversation or approval""></a></h2><p>Several <code>t.send</code> calls in one case share a session. Assertions on a returned turn inspect only that turn, while assertions on <code>t</code> inspect the whole conversation.</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> first</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Inspect PR 42."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
|
|
92
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">first.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"inspect_pr"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
|
|
175
93
|
<span class="line"></span>
|
|
176
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">
|
|
177
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
178
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
179
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
180
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
181
|
-
<span class="line"><span style="--shiki-light:#
|
|
182
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
183
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
184
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
185
|
-
<span class="line"><span style="--shiki-light:#
|
|
186
|
-
<span class="line"><span style="--shiki-light:#
|
|
187
|
-
<span class="line"><span style="--shiki-light:#
|
|
188
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
189
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
|
|
190
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> id</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"update-rule"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
191
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> description</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"A repeated billing rule parks the AGENTS.md write."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
192
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> async </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(t) {</span></span>
|
|
193
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Weekly AGENTS.md review. Read week/ and call apply_agents once."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, {</span></span>
|
|
194
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> workspaceFiles: {</span></span>
|
|
195
|
-
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "week/prs.md"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">RULE</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
196
|
-
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "week/slack.md"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">SLACK</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
197
|
-
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "week/tree/AGENTS.md.txt"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"# API</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\n\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">Keep handlers thin.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
198
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
|
|
199
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> });</span></span>
|
|
200
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">parked</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
|
|
201
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"apply_agents"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, { input: { verdict: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"update"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> } });</span></span>
|
|
94
|
+
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">const</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> followUp</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> =</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Now summarize only the blockers."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
|
|
95
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">followUp.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">usedNoTools</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
|
|
96
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(followUp.message, </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">includes</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#032F62;--shiki-dark:#DBEDFF;">blocker</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">i</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">));</span></span>
|
|
97
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span></code></pre></div><p>For a tool that requires approval, the expected result is a parked turn instead of a completed one:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Apply the approved policy update."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
|
|
98
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">parked</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
|
|
99
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">calledTool</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"apply_policy"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, {</span></span>
|
|
100
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> input: { verdict: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"update"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
|
|
101
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p>Pair an approval case with a safe branch that calls <code>t.notCalledTool("apply_policy")</code>, so both sides of the decision stay protected.</p><h2 id="pin-realistic-inputs" tabindex="-1">Pin realistic inputs <a class="header-anchor" href="#pin-realistic-inputs" aria-label="Permalink to "Pin realistic inputs""></a></h2><p>A useful eval changes only when the agent changes. Freeze canonical chat prompts, seed workspace evidence directly, and save webhook payloads instead of depending on a developer's checkout or live external state.</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Review pr/diff.patch and report blockers."</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, {</span></span>
|
|
102
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> workspaceFiles: {</span></span>
|
|
103
|
+
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "pr/diff.patch"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
|
|
104
|
+
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "diff --git a/app/routes/search.ts b/app/routes/search.ts"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
105
|
+
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "+res.send(`<h1>Results for ${req.query.q}</h1>`);"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
106
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> ].</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">join</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">\n</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">),</span></span>
|
|
202
107
|
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
|
|
203
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}
|
|
204
|
-
<span class="line"><span style="--shiki-light:#
|
|
205
|
-
<span class="line"><span style="--shiki-light:#
|
|
206
|
-
<span class="line"></span>
|
|
207
|
-
<span class="line"><span style="--shiki-light:#
|
|
208
|
-
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">
|
|
209
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span>
|
|
210
|
-
<span class="line"></span>
|
|
211
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> rows.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">map</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">((</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">row</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=></span></span>
|
|
212
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEval</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
|
|
213
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> description: row.task,</span></span>
|
|
214
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> async</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> test</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">t</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
|
|
215
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">send</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(row.prompt);</span></span>
|
|
216
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">succeeded</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">();</span></span>
|
|
217
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> t.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">check</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(t.reply, </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">equals</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(row.sql));</span></span>
|
|
218
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
|
|
219
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> })</span></span>
|
|
220
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">);</span></span></code></pre></div><h2 id="configure-eval-runs" tabindex="-1">Configure eval runs <a class="header-anchor" href="#configure-eval-runs" aria-label="Permalink to "Configure eval runs""></a></h2><p><code>evals/evals.config.ts</code> (or <code>.js</code>) holds project-wide defaults. It must set <code>maxConcurrency</code>. Each case issues real model requests, so concurrency is hard-capped at 200; the templates use 10. <code>eval --list</code> works without the file. Running a case does not.</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">import</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { defineEvalConfig } </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">from</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "@cursor/july/evals"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">;</span></span>
|
|
221
|
-
<span class="line"></span>
|
|
222
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">export</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> default</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> defineEvalConfig</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({</span></span>
|
|
223
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> maxConcurrency: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">20</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
224
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> timeoutMs: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">180_000</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
225
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> judge: { model: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"gpt-5.4-mini"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
|
|
226
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><table tabindex="0"><thead><tr><th>Option</th><th>Default</th><th>Meaning</th></tr></thead><tbody><tr><td><code>maxConcurrency</code></td><td>required</td><td>Datapoints in flight at once, 1-200</td></tr><tr><td><code>timeoutMs</code></td><td><code>180_000</code></td><td>Per-case timeout. Precedence: case or file <code>timeoutMs</code>, then <code>--timeout-ms</code>, then this value</td></tr><tr><td><code>judge</code></td><td>unset</td><td>Default judge model for <code>t.judge.*</code></td></tr><tr><td><code>reporters</code></td><td>unset</td><td>Destinations that observe every case; <code>--skip-report</code> suppresses them</td></tr><tr><td><code>maxPlaygroundRuns</code></td><td><code>20</code></td><td>Batches kept in playground history (cap 500). Counts <code>--prod</code> and <code>--url</code> batches, not the default local run</td></tr></tbody></table><p>Reporters ship results somewhere; the runner still does the grading. <code>JUnit({ filePath, suiteName? })</code> writes JUnit XML and <code>Artifacts({ dir })</code> writes per-case files, both from <code>@cursor/july/evals/reporters</code>. A custom reporter is an object with any of <code>onRunStart</code>, <code>onEvalComplete</code>, and <code>onRunComplete</code>. A reporter that throws is logged and never fails the run. CI usually attaches the built-in two with <code>--junit</code> and <code>--artifacts</code> instead of <code>reporters</code>, so output paths stay with the pipeline, not the eval author.</p><p>Playground batches survive restarts when the project configures durable storage. Otherwise they live in process memory until <code>serve</code> exits.</p><h2 id="run-evals-from-the-cli" tabindex="-1">Run evals from the CLI <a class="header-anchor" href="#run-evals-from-the-cli" aria-label="Permalink to "Run evals from the CLI""></a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --list</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # discover only</span></span>
|
|
227
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # run all</span></span>
|
|
228
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> builds/checkout</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # one datapoint</span></span>
|
|
229
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> builds</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> search</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # several ids or prefixes</span></span>
|
|
230
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> smoke</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> pull-request</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # any matching tag</span></span>
|
|
231
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --verbose</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # t.log lines + reply snippets</span></span>
|
|
232
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --timeout-ms</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> 300000</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # override the per-case timeout</span></span></code></pre></div><p>Id filters use OR semantics. Each filter selects an exact id and its descendants: <code>builds</code> selects <code>builds</code>, <code>builds/checkout</code>, and every other case below that path. Repeated tags also use OR. With both ids and tags, a case must match both groups.</p><p>By default <code>eval</code> boots a throwaway server with its own state root, so cases don't inherit your checkout's <code>AGENTS.md</code> and session state stays out of the project. Artifacts still land in the project state directory; see <a href="#where-results-land">Where results land</a>. <code>--slug</code> picks the target in a multi-agent directory.</p><p><code>--url</code> runs the batch on a running server instead, the same way <code>--prod</code> does: that server discovers its own <code>evals/</code>, results land in its playground history, and the local-only flags (<code>--junit</code>, <code>--artifacts</code>, <code>--max-concurrency</code>, <code>--skip-report</code>) do not apply. See <a href="#run-evals-in-the-playground-or-on-a-deployment">Run evals in the playground or on a deployment</a>.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --url</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> http://127.0.0.1:3000/weather-agent</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
|
|
233
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --bearer-token</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">$AGENT_TOKEN</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"</span></span></code></pre></div><p>See <a href="./reference/cli.html#eval">CLI: eval</a> for every flag.</p><h3 id="credentials" tabindex="-1">Credentials <a class="header-anchor" href="#credentials" aria-label="Permalink to "Credentials""></a></h3><p>Model turns need a <a href="./reference/cli.html#environment-variables">Cursor credential</a>: <code>CURSOR_API_KEY</code>, <code>CURSOR_SERVICE_ACCOUNT_KEY</code>, or a saved <code>agent-sdk login</code>. The judge uses the same one. <code>eval --list</code> needs none.</p><h3 id="where-results-land" tabindex="-1">Where results land <a class="header-anchor" href="#where-results-land" aria-label="Permalink to "Where results land""></a></h3><p>Every local run writes artifacts to a timestamped directory under <code>evals/</code> in the project state directory, whatever <code>--state-root</code> says. <code>--artifacts <dir></code> chooses the path and <code>--no-artifacts</code> skips them. The directory holds <code>summary.json</code>, <code>results.jsonl</code>, and <code>evals/<case-id>.json</code> with every assertion, the inputs, tool calls with arguments and output, the final text, and <code>t.log</code> lines. Start there when a case fails. <code>--out <file></code> also writes the full results JSON to a path of your choice.</p><p>The artifact does not include the session's event stream. Pass <code>--state-root <path></code> to keep the ephemeral server's <a href="./reference/sessions.html#where-does-the-agent-sdk-store-session-data">session data</a> on disk when you need the raw events.</p><h2 id="run-evals-in-ci" tabindex="-1">Run evals in CI <a class="header-anchor" href="#run-evals-in-ci" aria-label="Permalink to "Run evals in CI""></a></h2><p>Run the suite non-interactively, write JUnit for the CI annotations, and fail the job on a red gate:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># CURSOR_API_KEY comes from the CI secret store</span></span>
|
|
108
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">});</span></span></code></pre></div><p>For GitHub agents, snapshot a PR into committed fixtures:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> github</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> replay</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> acme/checkout#42</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
|
|
109
|
+
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --events</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> '*'</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
|
|
110
|
+
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dry-run</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
|
|
111
|
+
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --out</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> fixtures/github</span></span></code></pre></div><p>Keep a small, fast <code>smoke</code> set for every change. Put larger datasets and drift-prone live cases behind an explicit selection. See <a href="./reference/evals.html#datasets-and-fixtures">Datasets and fixtures</a> for loaders and case expansion.</p><h2 id="run-and-debug-a-regression-suite" tabindex="-1">Run and debug a regression suite <a class="header-anchor" href="#run-and-debug-a-regression-suite" aria-label="Permalink to "Run and debug a regression suite""></a></h2><p>Run one case while iterating, then the smoke suite before review. A failure artifact records the input, assertions, tool calls, and final text, making it the first place to look before changing the prompt.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> readiness</span></span>
|
|
112
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> smoke</span></span>
|
|
113
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span></span></code></pre></div><p>By default, results land under <code>evals/<stamp>/</code> in the project state directory. Open <code>evals/<case-id>.json</code> inside that run directory for the failed case. This artifact location is independent of <code>--state-root</code>; pass <code>--artifacts <dir></code> to choose another destination.</p><p>Model turns need a credential. Resolution checks <code>CURSOR_API_KEY</code>, <code>CURSOR_API_KEY_FILE</code>, <code>CURSOR_SERVICE_ACCOUNT_KEY</code>, then a saved <code>agent-sdk login</code>. <code>eval --list</code> only discovers cases and needs none.</p><h2 id="run-evals-in-ci" tabindex="-1">Run evals in CI <a class="header-anchor" href="#run-evals-in-ci" aria-label="Permalink to "Run evals in CI""></a></h2><p>Write machine-readable output and JUnit annotations, then let a failed gate fail the job:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># CURSOR_API_KEY comes from the CI secret store.</span></span>
|
|
234
114
|
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --json</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --no-stream</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
|
|
235
115
|
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --junit</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> reports/evals.xml</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
|
|
236
116
|
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --artifacts</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> reports/evals</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
|
|
237
|
-
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> ></span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> reports/evals.json</span></span></code></pre></div><p>
|
|
238
|
-
<span class="line"><span style="--shiki-light:#
|
|
239
|
-
<span class="line"><span style="--shiki-light:#
|
|
240
|
-
<span class="line"><span style="--shiki-light:#
|
|
241
|
-
<span class="line"><span style="--shiki-light:#
|
|
242
|
-
<span class="line"><span style="--shiki-light:#
|
|
243
|
-
<span class="line"><span style="--shiki-light:#
|
|
244
|
-
<span class="line"><span style="--shiki-light:#
|
|
245
|
-
<span class="line"
|
|
246
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">
|
|
247
|
-
<span class="line"><span style="--shiki-light:#
|
|
248
|
-
<
|
|
249
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "ok"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
250
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "assertions"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
|
|
251
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">"name"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"succeeded"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">"passed"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
|
|
252
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">"name"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"calledTool(inspect_pr)"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">"passed"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
|
|
253
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> ],</span></span>
|
|
254
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "sessionId"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"ses_123"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
255
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "inputs"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Is https://github.com/acme/checkout/pull/42 ready to approve?"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">],</span></span>
|
|
256
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "toolCalls"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">"toolName"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"inspect_pr"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">"isError"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">false</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }],</span></span>
|
|
257
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "metrics"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {},</span></span>
|
|
258
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "logs"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [],</span></span>
|
|
259
|
-
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "durationMs"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">12340</span></span>
|
|
260
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
|
|
261
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> ]</span></span>
|
|
262
|
-
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Each result can also include <code>description</code>, <code>finalText</code>, <code>tools</code>, <code>error</code>, <code>skipReason</code>, <code>metadata</code>, <code>tags</code>, and tool <code>args</code> and <code>output</code>. A soft miss shows as <code>"severity": "soft"</code> with <code>score</code> and <code>threshold</code> on the assertion. This shape lets CI report the failed assertion without parsing terminal text.</p><p>Keep CI green without weakening gates:</p><ul><li>Run <code>--tag smoke</code> on every push and the full suite on a schedule.</li><li>For probabilistic behavior, use <code>iterations</code> and a soft bar instead of one hard gate.</li></ul><h2 id="run-evals-in-the-playground-or-on-a-deployment" tabindex="-1">Run evals in the playground or on a deployment <a class="header-anchor" href="#run-evals-in-the-playground-or-on-a-deployment" aria-label="Permalink to "Run evals in the playground or on a deployment""></a></h2><p>Start the server, open the playground, and choose <strong>Evals</strong>. Run every case or one case, watch progress, and open the resulting session trace. Playground runs target the live server instead of an ephemeral one, so their sessions appear in the session list. One batch runs at a time.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> serve</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --dir</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> .</span></span></code></pre></div><p><code>--prod</code> (or <code>--url</code>) starts the same server-side batch on the team's hosted deployment (or the server you name), so results land in that server's playground history:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --tag</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> deepsec</span></span>
|
|
263
|
-
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Eval ID: <evalId></span></span>
|
|
264
|
-
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Playground: <deployment>/playground?view=evals&evalRunId=<evalId></span></span>
|
|
265
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> status</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> <</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">evalI</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">d</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">></span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span></span>
|
|
266
|
-
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">agent-sdk</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> eval</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> cancel</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> <</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">evalI</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">d</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">></span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --prod</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --slug</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> vulnerability-scanner</span></span></code></pre></div><p>The CLI prints the Eval ID as soon as the batch is accepted. Pass <code>--no-wait</code> to return right away and poll with <code>eval status</code> later; it exits <code>3</code> while the batch is still running. Hosted history follows <code>maxPlaygroundRuns</code> and the persistence rule under <a href="#configure-eval-runs">Configure eval runs</a>. The HTTP surface is under <a href="./reference/http-api.html#playground-eval-routes">Playground eval routes</a>.</p><h2 id="keep-improvements-with-regression-evals" tabindex="-1">Keep improvements with regression evals <a class="header-anchor" href="#keep-improvements-with-regression-evals" aria-label="Permalink to "Keep improvements with regression evals""></a></h2><p>Every <a href="./hillclimbing.html">hillclimb</a> round that keeps a change must land an eval that would have failed before the change. If you can't express the improvement as a gate (a <code>calledTool</code> shift, a bounded <code>maxToolCalls</code>, an output-shape check), the improvement is unverified, and it'll regress silently.</p><p>The rule cuts the other way too: never weaken an existing gate to make a round pass. That's the freeze line moving, and it turns your regression suite into a list of checks that no longer protect anything.</p><h2 id="what-s-next" tabindex="-1">What's next <a class="header-anchor" href="#what-s-next" aria-label="Permalink to "What's next""></a></h2><p>Continue with these pages:</p><ul><li><a href="./hillclimbing.html">Hillclimbing</a>: the loop evals make trustworthy</li><li><a href="./building-with-agents.html">Building agents with agents</a>: have a coding agent write the first suite</li><li><a href="./guides/github.html">GitHub guide</a>: deterministic webhook fixtures with <code>github replay</code></li><li><a href="./reference/sessions.html">Sessions and streaming</a>: the events <code>t.events</code> contains</li></ul></div></div></main><footer class="VPDocFooter" data-v-afe580da data-v-feab96e0><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-feab96e0><span class="visually-hidden" id="doc-footer-aria-label" data-v-feab96e0>Pager</span><div class="pager" data-v-feab96e0><a class="VPLink link pager-link prev" href="/docs/deployment.html" data-v-feab96e0><!--[--><span class="desc" data-v-feab96e0>Previous page</span><span class="title" data-v-feab96e0>Deployment</span><!--]--></a></div><div class="pager" data-v-feab96e0><a class="VPLink link pager-link next" href="/docs/hillclimbing.html" data-v-feab96e0><!--[--><span class="desc" data-v-feab96e0>Next page</span><span class="title" data-v-feab96e0>Hillclimbing</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
267
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"building-with-agents.md\":\"CEGVXkmO\",\"deployment.md\":\"Dm4Qo3hp\",\"evals.md\":\"BLDRt5LH\",\"guides_agent-to-agent.md\":\"BI0xclmy\",\"guides_bitbucket.md\":\"CTpCl__f\",\"guides_cloud-agents.md\":\"lSE_l7lH\",\"guides_convert-automation.md\":\"Ck6Cr68A\",\"guides_github.md\":\"D6ER29dG\",\"guides_gitlab.md\":\"P-TjBnS5\",\"guides_grokbot-agents.md\":\"WBZIOvkz\",\"guides_improve.md\":\"BKaDuKKK\",\"guides_mcp-oauth.md\":\"DMNMpXtO\",\"guides_opentelemetry.md\":\"_CRfDyzH\",\"guides_slack.md\":\"DdT8rmsj\",\"guides_webhooks.md\":\"aQW10HRe\",\"hillclimbing.md\":\"Dq4kkVIL\",\"index.md\":\"BFVyY2KT\",\"quickstart.md\":\"D3MjSZN-\",\"reference_agent-config.md\":\"CfVA-LZJ\",\"reference_artifacts.md\":\"Vf7qyIZ-\",\"reference_channels.md\":\"icqLKcTc\",\"reference_cli.md\":\"B2dBL6L8\",\"reference_connections.md\":\"Cb3U_c8n\",\"reference_extensions.md\":\"wlFD3cUR\",\"reference_hooks.md\":\"CZuynAxj\",\"reference_http-api.md\":\"DTKcYE6L\",\"reference_instructions.md\":\"D7gkckK-\",\"reference_playground.md\":\"D2YExv5K\",\"reference_project-layout.md\":\"DPxbUJyt\",\"reference_prompt.md\":\"BQ5uAv1F\",\"reference_schedules.md\":\"BasfZWO-\",\"reference_sessions.md\":\"YKvIsWAx\",\"reference_skills.md\":\"rNgpsGd0\",\"reference_subagents.md\":\"e5qitjJt\",\"reference_tools.md\":\"BdCO2aHZ\",\"templates_agentic-owners.md\":\"Da_AGDlH\",\"templates_pr-autofixer.md\":\"DqxocIGh\",\"templates_security-reviewer.md\":\"Bhnvd8VE\",\"templates_triage.md\":\"BdBWO9Ic\",\"troubleshooting.md\":\"BcgNoYtJ\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"Agent SDK\",\"description\":\"Filesystem-first framework for building and serving Cursor agents.\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"sdkVersion\":\"0.1.111\",\"nav\":[{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Guides\",\"link\":\"/guides/mcp-oauth\"},{\"text\":\"Integrations\",\"link\":\"/guides/github\"},{\"text\":\"Templates\",\"link\":\"/templates/security-reviewer\"},{\"text\":\"Reference\",\"link\":\"/reference/project-layout\"}],\"search\":{\"provider\":\"local\"},\"outline\":{\"level\":[2,3]},\"sidebar\":[{\"text\":\"Getting started\",\"items\":[{\"text\":\"Overview\",\"link\":\"/\"},{\"text\":\"Building your first agent\",\"link\":\"/quickstart\"}]},{\"text\":\"Integrations\",\"collapsed\":false,\"items\":[{\"text\":\"GitHub\",\"link\":\"/guides/github\"},{\"text\":\"Slack\",\"link\":\"/guides/slack\"},{\"text\":\"Webhooks\",\"link\":\"/guides/webhooks\"},{\"text\":\"Cursor Automations\",\"link\":\"/guides/convert-automation\"},{\"text\":\"OpenTelemetry\",\"link\":\"/guides/opentelemetry\"},{\"text\":\"GitLab\",\"link\":\"/guides/gitlab\"},{\"text\":\"Bitbucket\",\"link\":\"/guides/bitbucket\"}]},{\"text\":\"Extensions\",\"collapsed\":false,\"items\":[{\"text\":\"Overview\",\"link\":\"/reference/extensions\"},{\"text\":\"Cloud Agents\",\"link\":\"/guides/cloud-agents\"},{\"text\":\"Grok Bot Agents\",\"link\":\"/guides/grokbot-agents\"},{\"text\":\"Self-improving Agents\",\"link\":\"/guides/improve\"}]},{\"text\":\"Guides\",\"collapsed\":false,\"items\":[{\"text\":\"MCP\",\"link\":\"/guides/mcp-oauth\"},{\"text\":\"Agent-to-agent\",\"link\":\"/guides/agent-to-agent\"},{\"text\":\"Deployment\",\"link\":\"/deployment\"},{\"text\":\"Evals\",\"link\":\"/evals\"},{\"text\":\"Hillclimbing\",\"link\":\"/hillclimbing\"},{\"text\":\"Self Improving Agents\",\"link\":\"/building-with-agents\"},{\"text\":\"Troubleshooting\",\"link\":\"/troubleshooting\"}]},{\"text\":\"Templates\",\"collapsed\":false,\"items\":[{\"text\":\"Security reviewer\",\"link\":\"/templates/security-reviewer\"},{\"text\":\"Agentic Approver\",\"link\":\"/templates/agentic-owners\"},{\"text\":\"PR autofixer\",\"link\":\"/templates/pr-autofixer\"},{\"text\":\"Triage agent\",\"link\":\"/templates/triage\"}]},{\"text\":\"Reference\",\"collapsed\":false,\"items\":[{\"text\":\"Project layout\",\"link\":\"/reference/project-layout\"},{\"text\":\"Agent config\",\"link\":\"/reference/agent-config\"},{\"text\":\"Instructions\",\"link\":\"/reference/instructions\"},{\"text\":\"Tools\",\"link\":\"/reference/tools\"},{\"text\":\"prompt\",\"link\":\"/reference/prompt\"},{\"text\":\"Skills\",\"link\":\"/reference/skills\"},{\"text\":\"MCP Connections\",\"link\":\"/reference/connections\"},{\"text\":\"Subagents\",\"link\":\"/reference/subagents\"},{\"text\":\"Channels\",\"link\":\"/reference/channels\"},{\"text\":\"Schedules & reminders\",\"link\":\"/reference/schedules\"},{\"text\":\"Hooks\",\"link\":\"/reference/hooks\"},{\"text\":\"Artifacts\",\"link\":\"/reference/artifacts\"},{\"text\":\"Sessions & streaming\",\"link\":\"/reference/sessions\"},{\"text\":\"Playground\",\"link\":\"/reference/playground\"},{\"text\":\"CLI\",\"link\":\"/reference/cli\"},{\"text\":\"HTTP API\",\"link\":\"/reference/http-api\"}]}]},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
|
|
117
|
+
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> ></span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> reports/evals.json</span></span></code></pre></div><p>Run <code>--tag smoke</code> on pull requests and the full suite on a schedule when cost or latency makes every-push coverage impractical. Use soft scores for a new probabilistic benchmark until its threshold is trustworthy.</p><p>The <a href="./reference/cli.html#eval">CLI reference</a> covers selectors, hosted runs, exit codes, and JSON output.</p><h2 id="keep-side-effects-out-of-eval-sessions" tabindex="-1">Keep side effects out of eval sessions <a class="header-anchor" href="#keep-side-effects-out-of-eval-sessions" aria-label="Permalink to "Keep side effects out of eval sessions""></a></h2><p>Eval sessions carry <code>purpose: "eval"</code>. Check it at the deterministic write boundary and return the decision without performing the external action:</p><div class="language-ts vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">ts</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">async </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">execute</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">({ findings }, ctx) {</span></span>
|
|
118
|
+
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> if</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> (ctx.session.purpose </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">===</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "eval"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) {</span></span>
|
|
119
|
+
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> {</span></span>
|
|
120
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> posted: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">false</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
121
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> reason: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"eval"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
122
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> count: findings.</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">length</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
|
123
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> };</span></span>
|
|
124
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
|
|
125
|
+
<span class="line"></span>
|
|
126
|
+
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> return</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> await</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> postFindings</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(findings);</span></span>
|
|
127
|
+
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>This preserves the tool call in the trajectory, so the eval can still assert that the agent chose to post.</p><h2 id="evals-hooks-or-hillclimbing" tabindex="-1">Evals, hooks, or hillclimbing? <a class="header-anchor" href="#evals-hooks-or-hillclimbing" aria-label="Permalink to "Evals, hooks, or hillclimbing?""></a></h2><p>Use an eval to gate one fixed input. Use a <a href="./reference/hooks.html">hook</a> to observe every live session for metrics, auditing, or alerts. Use <a href="./hillclimbing.html">hillclimbing</a> to decide which source change improves a fixed set of inputs; every kept hillclimb change should add an eval.</p><h2 id="gates-and-scores" tabindex="-1">Gates and scores <a class="header-anchor" href="#gates-and-scores" aria-label="Permalink to "Gates and scores""></a></h2><p><code>t.succeeded()</code>, tool assertions, and deterministic checks are hard gates by default. A failed gate fails the case. <code>.soft()</code> records a measurement without blocking, while <code>.atLeast(threshold)</code> records a score and marks a miss as <code>scored</code>.</p><p>Start with decisions and output shape: the intended tool, the tempting wrong tool, and one structural check. Exact prose is rarely a stable contract.</p><h2 id="keep-improvements-with-regression-evals" tabindex="-1">Keep improvements with regression evals <a class="header-anchor" href="#keep-improvements-with-regression-evals" aria-label="Permalink to "Keep improvements with regression evals""></a></h2><p>Every kept improvement needs an eval that would have failed before the change. A tool-choice gate, bounded <code>maxToolCalls</code>, or output-shape check turns the improvement into a durable contract.</p><p>Never weaken an existing gate to make a new implementation pass. That moves the freeze line instead of proving the change.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to "Related""></a></h2><ul><li><a href="./reference/evals.html">Evals reference</a>: complete authoring and runner contracts</li><li><a href="./../skills/evals/SKILL.html"><code>skills/evals/SKILL.md</code></a>: author and seed cases with a coding agent</li><li><a href="./hillclimbing.html">Hillclimbing</a>: measure a change and lock the win</li><li><a href="./reference/sessions.html">Sessions</a>: events available to trajectory assertions</li></ul></div></div></main><footer class="VPDocFooter" data-v-afe580da data-v-feab96e0><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-feab96e0><span class="visually-hidden" id="doc-footer-aria-label" data-v-feab96e0>Pager</span><div class="pager" data-v-feab96e0><a class="VPLink link pager-link prev" href="/docs/deployment.html" data-v-feab96e0><!--[--><span class="desc" data-v-feab96e0>Previous page</span><span class="title" data-v-feab96e0>Deployment</span><!--]--></a></div><div class="pager" data-v-feab96e0><a class="VPLink link pager-link next" href="/docs/hillclimbing.html" data-v-feab96e0><!--[--><span class="desc" data-v-feab96e0>Next page</span><span class="title" data-v-feab96e0>Hillclimbing</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
128
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"building-with-agents.md\":\"BBCx0AUo\",\"deployment.md\":\"D2jQZuFx\",\"evals.md\":\"D3Y3Aixt\",\"guides_agent-to-agent.md\":\"CD4T5FIl\",\"guides_bitbucket.md\":\"mpevW-VP\",\"guides_cloud-agents.md\":\"Cp1O3u-X\",\"guides_convert-automation.md\":\"CqEyfP6Y\",\"guides_github.md\":\"BwpBp3ed\",\"guides_gitlab.md\":\"DaEC3nMk\",\"guides_grokbot-agents.md\":\"CMhZNdEU\",\"guides_improve.md\":\"Bnp4F99w\",\"guides_jev.md\":\"F5fAkkfN\",\"guides_mcp-oauth.md\":\"bSFakfCY\",\"guides_opentelemetry.md\":\"BKDxQmmd\",\"guides_slack.md\":\"Bo96y42E\",\"guides_webhooks.md\":\"1A72_VEE\",\"hillclimbing.md\":\"D4E1o5Sa\",\"index.md\":\"C-t81M5J\",\"quickstart.md\":\"DAvVhuuU\",\"reference_agent-config.md\":\"DGPyw7ms\",\"reference_artifacts.md\":\"Bu_4HmsD\",\"reference_channels.md\":\"nFWbzAic\",\"reference_cli.md\":\"DLWDz9ij\",\"reference_connections.md\":\"Je9dMsdd\",\"reference_evals.md\":\"DNJzM_yf\",\"reference_extensions.md\":\"Cv5aLCz_\",\"reference_hooks.md\":\"B7uzNENk\",\"reference_http-api.md\":\"CduHavZ2\",\"reference_instructions.md\":\"CU1My5My\",\"reference_playground.md\":\"Ch2d0Iqi\",\"reference_project-layout.md\":\"BGhgpy9V\",\"reference_prompt.md\":\"Ccp0R53H\",\"reference_schedules.md\":\"B2Nm6FaD\",\"reference_sessions.md\":\"1_6Vyv7x\",\"reference_skills.md\":\"DjQkRefx\",\"reference_subagents.md\":\"Dl16gcBj\",\"reference_tools.md\":\"B1dH1lpa\",\"templates_agentic-owners.md\":\"9M575F5C\",\"templates_pr-autofixer.md\":\"ws0DDXDy\",\"templates_security-reviewer.md\":\"KEFYXzfK\",\"templates_thermo-quality-review.md\":\"VNJ_mohX\",\"templates_thermo-review.md\":\"Hi3zWOkP\",\"templates_triage.md\":\"CVGe_FG6\",\"troubleshooting.md\":\"mnfFG2Em\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"Agent SDK\",\"description\":\"Filesystem-first framework for building and serving Cursor agents.\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"sdkVersion\":\"0.1.113\",\"nav\":[{\"text\":\"Quickstart\",\"link\":\"/quickstart\"},{\"text\":\"Guides\",\"link\":\"/guides/mcp-oauth\"},{\"text\":\"Integrations\",\"link\":\"/guides/github\"},{\"text\":\"Templates\",\"link\":\"/templates/security-reviewer\"},{\"text\":\"Reference\",\"link\":\"/reference/project-layout\"}],\"search\":{\"provider\":\"local\"},\"outline\":{\"level\":[2,3]},\"sidebar\":[{\"text\":\"Getting started\",\"items\":[{\"text\":\"Overview\",\"link\":\"/\"},{\"text\":\"Building your first agent\",\"link\":\"/quickstart\"}]},{\"text\":\"Integrations\",\"collapsed\":false,\"items\":[{\"text\":\"GitHub\",\"link\":\"/guides/github\"},{\"text\":\"Slack\",\"link\":\"/guides/slack\"},{\"text\":\"Webhooks\",\"link\":\"/guides/webhooks\"},{\"text\":\"Cursor Automations\",\"link\":\"/guides/convert-automation\"},{\"text\":\"OpenTelemetry\",\"link\":\"/guides/opentelemetry\"},{\"text\":\"GitLab\",\"link\":\"/guides/gitlab\"},{\"text\":\"Bitbucket\",\"link\":\"/guides/bitbucket\"}]},{\"text\":\"Extensions\",\"collapsed\":false,\"items\":[{\"text\":\"Overview\",\"link\":\"/reference/extensions\"},{\"text\":\"Cloud agents\",\"link\":\"/guides/cloud-agents\"},{\"text\":\"Grok Bot agents\",\"link\":\"/guides/grokbot-agents\"},{\"text\":\"Self-improvement\",\"link\":\"/guides/improve\"},{\"text\":\"Jev\",\"link\":\"/guides/jev\"}]},{\"text\":\"Guides\",\"collapsed\":false,\"items\":[{\"text\":\"MCP\",\"link\":\"/guides/mcp-oauth\"},{\"text\":\"Peer agents\",\"link\":\"/guides/agent-to-agent\"},{\"text\":\"Deployment\",\"link\":\"/deployment\"},{\"text\":\"Evals\",\"link\":\"/evals\"},{\"text\":\"Hillclimbing\",\"link\":\"/hillclimbing\"},{\"text\":\"Build with a coding agent\",\"link\":\"/building-with-agents\"},{\"text\":\"Troubleshooting\",\"link\":\"/troubleshooting\"}]},{\"text\":\"Templates\",\"collapsed\":false,\"items\":[{\"text\":\"Security reviewer\",\"link\":\"/templates/security-reviewer\"},{\"text\":\"Thermo review\",\"link\":\"/templates/thermo-review\"},{\"text\":\"Thermo quality review\",\"link\":\"/templates/thermo-quality-review\"},{\"text\":\"Agentic Approver\",\"link\":\"/templates/agentic-owners\"},{\"text\":\"PR autofixer\",\"link\":\"/templates/pr-autofixer\"},{\"text\":\"Triage agent\",\"link\":\"/templates/triage\"}]},{\"text\":\"Reference\",\"collapsed\":false,\"items\":[{\"text\":\"Project layout\",\"link\":\"/reference/project-layout\"},{\"text\":\"Agent config\",\"link\":\"/reference/agent-config\"},{\"text\":\"Instructions\",\"link\":\"/reference/instructions\"},{\"text\":\"Tools\",\"link\":\"/reference/tools\"},{\"text\":\"prompt\",\"link\":\"/reference/prompt\"},{\"text\":\"Skills\",\"link\":\"/reference/skills\"},{\"text\":\"MCP Connections\",\"link\":\"/reference/connections\"},{\"text\":\"Subagents\",\"link\":\"/reference/subagents\"},{\"text\":\"Channels\",\"link\":\"/reference/channels\"},{\"text\":\"Schedules & reminders\",\"link\":\"/reference/schedules\"},{\"text\":\"Hooks\",\"link\":\"/reference/hooks\"},{\"text\":\"Evals\",\"link\":\"/reference/evals\"},{\"text\":\"Artifacts\",\"link\":\"/reference/artifacts\"},{\"text\":\"Sessions & streaming\",\"link\":\"/reference/sessions\"},{\"text\":\"Playground\",\"link\":\"/reference/playground\"},{\"text\":\"CLI\",\"link\":\"/reference/cli\"},{\"text\":\"HTTP API\",\"link\":\"/reference/http-api\"}]}]},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
|
|
268
129
|
|
|
269
130
|
</body>
|
|
270
131
|
</html>
|