@grok/agent-sdk 0.2.29 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channels/slack/cli-project.js +1 -1
- package/dist/docs/404.html +2 -2
- package/dist/docs/assets/{app.DMrZKFjI.js → app.BzTUBOAD.js} +4 -4
- package/dist/docs/assets/building-with-agents.md.BqJ7AAjG.js +6 -0
- package/dist/docs/assets/{building-with-agents.md.Bf1yH2tG.lean.js → building-with-agents.md.BqJ7AAjG.lean.js} +1 -1
- package/dist/docs/assets/changelog.md.BWhC6DET.js +1 -0
- package/dist/docs/assets/changelog.md.BWhC6DET.lean.js +1 -0
- package/dist/docs/assets/chunks/@localSearchIndexroot.CWxwVoCG.js +1 -0
- package/dist/docs/assets/chunks/{VPLocalSearchBox.DQvuw1KA.js → VPLocalSearchBox.DsnRlUT5.js} +1 -1
- package/dist/docs/assets/chunks/{arc.ahOG-Qza.js → arc.DH8GOhiv.js} +1 -1
- package/dist/docs/assets/chunks/{architectureDiagram-Q4EWVU46.BcgfYNO_.js → architectureDiagram-Q4EWVU46.9m9ObyuE.js} +1 -1
- package/dist/docs/assets/chunks/{baseUniq.DeIYZrWh.js → baseUniq.BXHpx1lE.js} +1 -1
- package/dist/docs/assets/chunks/{blockDiagram-DXYQGD6D.HjJ2goMX.js → blockDiagram-DXYQGD6D.xY3EX59q.js} +1 -1
- package/dist/docs/assets/chunks/{c4Diagram-AHTNJAMY.C1QTb9GM.js → c4Diagram-AHTNJAMY.DnqgMVgc.js} +1 -1
- package/dist/docs/assets/chunks/channel.Cctv0f7X.js +1 -0
- package/dist/docs/assets/chunks/{chunk-4BX2VUAB.UuZiDiVp.js → chunk-4BX2VUAB.BjFxK2A3.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-4TB4RGXK.5A3YmkhI.js → chunk-4TB4RGXK.BujS_Spt.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-55IACEB6.B4uf1yYV.js → chunk-55IACEB6.YbTjhdaM.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-EDXVE4YY.CoXBce8t.js → chunk-EDXVE4YY.DnQ8t4GC.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-FMBD7UC4.DnC5NO4e.js → chunk-FMBD7UC4.B67AwAmj.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-OYMX7WX6.CzfHCeIi.js → chunk-OYMX7WX6.DrZIAFOm.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-QZHKN3VN.C6tcwkVd.js → chunk-QZHKN3VN.BLmmydkn.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-YZCP3GAM._oboyZ1A.js → chunk-YZCP3GAM.DaoOfw0O.js} +1 -1
- package/dist/docs/assets/chunks/classDiagram-6PBFFD2Q.DU5sqchW.js +1 -0
- package/dist/docs/assets/chunks/classDiagram-v2-HSJHXN6E.DU5sqchW.js +1 -0
- package/dist/docs/assets/chunks/clone.B9rdNt6j.js +1 -0
- package/dist/docs/assets/chunks/{cose-bilkent-S5V4N54A.BfPVKeId.js → cose-bilkent-S5V4N54A.DMQuNhM5.js} +1 -1
- package/dist/docs/assets/chunks/{dagre-KV5264BT.CLMLc0lD.js → dagre-KV5264BT.CZoGVuUf.js} +1 -1
- package/dist/docs/assets/chunks/{diagram-5BDNPKRD.C7OF7UfH.js → diagram-5BDNPKRD.CLt9uOyL.js} +1 -1
- package/dist/docs/assets/chunks/{diagram-G4DWMVQ6.Dpao0Bl1.js → diagram-G4DWMVQ6.DpczEm1G.js} +1 -1
- package/dist/docs/assets/chunks/{diagram-MMDJMWI5.DLnM9eLp.js → diagram-MMDJMWI5.D4r13DvJ.js} +1 -1
- package/dist/docs/assets/chunks/{diagram-TYMM5635.BcpVgA_M.js → diagram-TYMM5635.CYU9eGFu.js} +1 -1
- package/dist/docs/assets/chunks/{erDiagram-SMLLAGMA.CsL5JuCV.js → erDiagram-SMLLAGMA.QA9naWEw.js} +1 -1
- package/dist/docs/assets/chunks/{flowDiagram-DWJPFMVM.MFihq9AF.js → flowDiagram-DWJPFMVM.BbNoVZYd.js} +1 -1
- package/dist/docs/assets/chunks/{ganttDiagram-T4ZO3ILL.CJeLodyG.js → ganttDiagram-T4ZO3ILL.IHjDoG7r.js} +1 -1
- package/dist/docs/assets/chunks/{gitGraphDiagram-UUTBAWPF.1HhlE45Z.js → gitGraphDiagram-UUTBAWPF.B21VMdf5.js} +1 -1
- package/dist/docs/assets/chunks/{graph.BS02KkWA.js → graph.DgJ4L-8V.js} +1 -1
- package/dist/docs/assets/chunks/{infoDiagram-42DDH7IO.fp5gvytQ.js → infoDiagram-42DDH7IO.CQjrcthc.js} +1 -1
- package/dist/docs/assets/chunks/{ishikawaDiagram-UXIWVN3A.C6Wk1V4I.js → ishikawaDiagram-UXIWVN3A.D4tPGtVo.js} +1 -1
- package/dist/docs/assets/chunks/{journeyDiagram-VCZTEJTY.qJt4uJ_H.js → journeyDiagram-VCZTEJTY.BD3WaNbQ.js} +1 -1
- package/dist/docs/assets/chunks/{kanban-definition-6JOO6SKY.z2tG9kly.js → kanban-definition-6JOO6SKY.Ca0o4Tqz.js} +1 -1
- package/dist/docs/assets/chunks/{layout.BPCkdejr.js → layout.DXXqX4MQ.js} +1 -1
- package/dist/docs/assets/chunks/{linear.CL3TC3u7.js → linear.5c61gNw4.js} +1 -1
- package/dist/docs/assets/chunks/{min.SPrSp704.js → min.B1-NAjHX.js} +1 -1
- package/dist/docs/assets/chunks/{mindmap-definition-QFDTVHPH.DgrQRFZW.js → mindmap-definition-QFDTVHPH.BELznv3-.js} +1 -1
- package/dist/docs/assets/chunks/{pieDiagram-DEJITSTG.Ba7yxiW2.js → pieDiagram-DEJITSTG.qPzO1mq5.js} +1 -1
- package/dist/docs/assets/chunks/{quadrantDiagram-34T5L4WZ.Bvskad4p.js → quadrantDiagram-34T5L4WZ.CxIvzbjX.js} +1 -1
- package/dist/docs/assets/chunks/{requirementDiagram-MS252O5E.Ce70ugV9.js → requirementDiagram-MS252O5E.CqA0iJx8.js} +1 -1
- package/dist/docs/assets/chunks/{sankeyDiagram-XADWPNL6.BJ5OEULO.js → sankeyDiagram-XADWPNL6.XRTutBnb.js} +1 -1
- package/dist/docs/assets/chunks/{sequenceDiagram-FGHM5R23.C8a1hDs6.js → sequenceDiagram-FGHM5R23.E--7bRLQ.js} +1 -1
- package/dist/docs/assets/chunks/{stateDiagram-FHFEXIEX.D2D8A5CK.js → stateDiagram-FHFEXIEX.C4vGu8zq.js} +1 -1
- package/dist/docs/assets/chunks/stateDiagram-v2-QKLJ7IA2.ECMu2BdJ.js +1 -0
- package/dist/docs/assets/chunks/{theme.DRciHmS1.js → theme.DMj_7q3B.js} +2 -2
- package/dist/docs/assets/chunks/{timeline-definition-GMOUNBTQ.C4ePrY8-.js → timeline-definition-GMOUNBTQ.C7LgNVAL.js} +1 -1
- package/dist/docs/assets/chunks/{vennDiagram-DHZGUBPP.Dzso6dEA.js → vennDiagram-DHZGUBPP.HUwoZcy_.js} +1 -1
- package/dist/docs/assets/chunks/{wardley-RL74JXVD.djgXR0ar.js → wardley-RL74JXVD.BxFEFYZS.js} +1 -1
- package/dist/docs/assets/chunks/{wardleyDiagram-NUSXRM2D.Bf2ICxjh.js → wardleyDiagram-NUSXRM2D.B2pSo5rB.js} +1 -1
- package/dist/docs/assets/chunks/{xychartDiagram-5P7HB3ND.DEH4BcVJ.js → xychartDiagram-5P7HB3ND.NQhMiOOL.js} +1 -1
- package/dist/docs/assets/deployment.md.BtqJrHhp.js +31 -0
- package/dist/docs/assets/{deployment.md.KW0Ni_rY.lean.js → deployment.md.BtqJrHhp.lean.js} +1 -1
- package/dist/docs/assets/evals.md.Dne6VRmZ.js +49 -0
- package/dist/docs/assets/evals.md.Dne6VRmZ.lean.js +1 -0
- package/dist/docs/assets/{guides_agent-to-agent.md.D1D5f6Jc.js → guides_agent-to-agent.md.BcmSqL2z.js} +8 -8
- package/dist/docs/assets/guides_agent-to-agent.md.BcmSqL2z.lean.js +1 -0
- package/dist/docs/assets/{guides_bitbucket.md.DkjkBRU8.js → guides_bitbucket.md.BiA6ZMFW.js} +27 -27
- package/dist/docs/assets/{guides_bitbucket.md.DkjkBRU8.lean.js → guides_bitbucket.md.BiA6ZMFW.lean.js} +1 -1
- package/dist/docs/assets/guides_cloud-agents.md.CiXAW6on.js +9 -0
- package/dist/docs/assets/{guides_cloud-agents.md.Ceyjv0ul.lean.js → guides_cloud-agents.md.CiXAW6on.lean.js} +1 -1
- package/dist/docs/assets/guides_convert-automation.md.BvTxsJ8J.js +26 -0
- package/dist/docs/assets/{guides_convert-automation.md.BjraCvok.lean.js → guides_convert-automation.md.BvTxsJ8J.lean.js} +1 -1
- package/dist/docs/assets/{guides_github.md.CgN5sNbW.js → guides_github.md.6jiEH7dg.js} +11 -19
- package/dist/docs/assets/{guides_github.md.CgN5sNbW.lean.js → guides_github.md.6jiEH7dg.lean.js} +1 -1
- package/dist/docs/assets/{guides_gitlab.md.DCE0p7_w.js → guides_gitlab.md.DcKusOcU.js} +63 -73
- package/dist/docs/assets/{guides_gitlab.md.DCE0p7_w.lean.js → guides_gitlab.md.DcKusOcU.lean.js} +1 -1
- package/dist/docs/assets/guides_improve.md.o3WOGR7n.js +16 -0
- package/dist/docs/assets/{guides_improve.md.BFYtrtLE.lean.js → guides_improve.md.o3WOGR7n.lean.js} +1 -1
- package/dist/docs/assets/{guides_jev.md.CEqLgjj9.js → guides_jev.md.DGCzkMUN.js} +20 -60
- package/dist/docs/assets/{guides_jev.md.CEqLgjj9.lean.js → guides_jev.md.DGCzkMUN.lean.js} +1 -1
- package/dist/docs/assets/{guides_mcp-oauth.md.DrGY92u4.js → guides_mcp-oauth.md.C-Sx_IPa.js} +3 -5
- package/dist/docs/assets/{guides_mcp-oauth.md.DrGY92u4.lean.js → guides_mcp-oauth.md.C-Sx_IPa.lean.js} +1 -1
- package/dist/docs/assets/guides_migrate.md.BbhHYeCr.js +3 -0
- package/dist/docs/assets/{guides_migrate.md.RS9Eu-Xh.lean.js → guides_migrate.md.BbhHYeCr.lean.js} +1 -1
- package/dist/docs/assets/guides_opentelemetry.md.BRSktBHt.js +26 -0
- package/dist/docs/assets/guides_opentelemetry.md.BRSktBHt.lean.js +1 -0
- package/dist/docs/assets/guides_pagerduty.md.pMXUkZZc.js +61 -0
- package/dist/docs/assets/guides_pagerduty.md.pMXUkZZc.lean.js +1 -0
- package/dist/docs/assets/guides_slack.md.BgIGG9ur.js +82 -0
- package/dist/docs/assets/{guides_slack.md.BwXHURvY.lean.js → guides_slack.md.BgIGG9ur.lean.js} +1 -1
- package/dist/docs/assets/{guides_webhooks.md.BJk63ZTs.js → guides_webhooks.md.B5DXt2v1.js} +7 -7
- package/dist/docs/assets/{guides_webhooks.md.BJk63ZTs.lean.js → guides_webhooks.md.B5DXt2v1.lean.js} +1 -1
- package/dist/docs/assets/hillclimbing.md.3TkdF79A.js +5 -0
- package/dist/docs/assets/hillclimbing.md.3TkdF79A.lean.js +1 -0
- package/dist/docs/assets/index.md.DRfaoRCZ.js +20 -0
- package/dist/docs/assets/{index.md.CAruw_rp.lean.js → index.md.DRfaoRCZ.lean.js} +1 -1
- package/dist/docs/assets/quickstart.md.r-oE8nkX.js +113 -0
- package/dist/docs/assets/{quickstart.md.CljDaJBi.lean.js → quickstart.md.r-oE8nkX.lean.js} +1 -1
- package/dist/docs/assets/reference_access.md.C5lblfUC.js +1 -0
- package/dist/docs/assets/reference_access.md.C5lblfUC.lean.js +1 -0
- package/dist/docs/assets/reference_agent-config.md.C0Tp_Fk9.js +27 -0
- package/dist/docs/assets/{reference_agent-config.md.Ck29qYNj.lean.js → reference_agent-config.md.C0Tp_Fk9.lean.js} +1 -1
- package/dist/docs/assets/reference_artifacts.md.BJpAI7h9.js +17 -0
- package/dist/docs/assets/{reference_artifacts.md.C-hKR2hp.lean.js → reference_artifacts.md.BJpAI7h9.lean.js} +1 -1
- package/dist/docs/assets/reference_channels.md.CPWOpdsC.js +21 -0
- package/dist/docs/assets/{reference_channels.md.DsJfGSBJ.lean.js → reference_channels.md.CPWOpdsC.lean.js} +1 -1
- package/dist/docs/assets/reference_cli.md.B8GdpaRj.js +83 -0
- package/dist/docs/assets/reference_cli.md.B8GdpaRj.lean.js +1 -0
- package/dist/docs/assets/reference_connections.md.DoY4vrDH.js +54 -0
- package/dist/docs/assets/{reference_connections.md.BoAotr7_.lean.js → reference_connections.md.DoY4vrDH.lean.js} +1 -1
- package/dist/docs/assets/reference_evals.md.DgfUtfIc.js +69 -0
- package/dist/docs/assets/{reference_evals.md.2BXeMiO2.lean.js → reference_evals.md.DgfUtfIc.lean.js} +1 -1
- package/dist/docs/assets/{reference_extensions.md.DI5X5x8p.js → reference_extensions.md.DHbW6eOf.js} +6 -8
- package/dist/docs/assets/{reference_extensions.md.DI5X5x8p.lean.js → reference_extensions.md.DHbW6eOf.lean.js} +1 -1
- package/dist/docs/assets/reference_hooks.md.CSWmOwMM.js +39 -0
- package/dist/docs/assets/reference_hooks.md.CSWmOwMM.lean.js +1 -0
- package/dist/docs/assets/reference_http-api.md.C7Cb0r2J.js +7 -0
- package/dist/docs/assets/{reference_http-api.md._RWELr3d.lean.js → reference_http-api.md.C7Cb0r2J.lean.js} +1 -1
- package/dist/docs/assets/reference_instructions.md.CV6Qq0UX.js +14 -0
- package/dist/docs/assets/reference_instructions.md.CV6Qq0UX.lean.js +1 -0
- package/dist/docs/assets/reference_limits.md.CbdtFr4T.js +8 -0
- package/dist/docs/assets/reference_limits.md.CbdtFr4T.lean.js +1 -0
- package/dist/docs/assets/reference_memory.md.CdoJISM-.js +3 -0
- package/dist/docs/assets/{reference_memory.md.Dd7cyV9V.lean.js → reference_memory.md.CdoJISM-.lean.js} +1 -1
- package/dist/docs/assets/reference_playground.md.DIUsRAcy.js +1 -0
- package/dist/docs/assets/{reference_playground.md.CQiJk9CM.lean.js → reference_playground.md.DIUsRAcy.lean.js} +1 -1
- package/dist/docs/assets/reference_project-layout.md.9DiKoE16.js +26 -0
- package/dist/docs/assets/reference_project-layout.md.9DiKoE16.lean.js +1 -0
- package/dist/docs/assets/reference_schedules.md.RhQ9ZLo1.js +45 -0
- package/dist/docs/assets/{reference_schedules.md.RC47aFm4.lean.js → reference_schedules.md.RhQ9ZLo1.lean.js} +1 -1
- package/dist/docs/assets/reference_sessions.md.NXVQt9mP.js +22 -0
- package/dist/docs/assets/{reference_sessions.md.C7KFhWXV.lean.js → reference_sessions.md.NXVQt9mP.lean.js} +1 -1
- package/dist/docs/assets/{reference_skills.md.Dej2zQxi.js → reference_skills.md.DzCPNPkG.js} +4 -4
- package/dist/docs/assets/{reference_skills.md.Dej2zQxi.lean.js → reference_skills.md.DzCPNPkG.lean.js} +1 -1
- package/dist/docs/assets/reference_subagents.md.Dvpq2sUu.js +10 -0
- package/dist/docs/assets/{reference_subagents.md.CAhvW4yS.lean.js → reference_subagents.md.Dvpq2sUu.lean.js} +1 -1
- package/dist/docs/assets/reference_tools.md.B2OaR1Nb.js +62 -0
- package/dist/docs/assets/{reference_tools.md.DQr7Fnhu.lean.js → reference_tools.md.B2OaR1Nb.lean.js} +1 -1
- package/dist/docs/assets/templates_agentic-owners.md.B6ZmKAzT.js +13 -0
- package/dist/docs/assets/{templates_agentic-owners.md.Dh2nDSqg.lean.js → templates_agentic-owners.md.B6ZmKAzT.lean.js} +1 -1
- package/dist/docs/assets/templates_reviews.md.dr9Xv_KM.js +10 -0
- package/dist/docs/assets/templates_reviews.md.dr9Xv_KM.lean.js +1 -0
- package/dist/docs/assets/troubleshooting.md.DfKKvW1R.js +3 -0
- package/dist/docs/assets/{troubleshooting.md.CGOdNVAL.lean.js → troubleshooting.md.DfKKvW1R.lean.js} +1 -1
- package/dist/docs/building-with-agents.html +41 -44
- package/dist/docs/building-with-agents.md +11 -104
- package/dist/docs/changelog.html +37 -37
- package/dist/docs/changelog.md +5 -155
- package/dist/docs/deployment.html +54 -54
- package/dist/docs/deployment.md +83 -262
- package/dist/docs/evals.html +52 -71
- package/dist/docs/evals.md +33 -129
- package/dist/docs/guides/agent-to-agent.html +43 -43
- package/dist/docs/guides/agent-to-agent.md +27 -64
- package/dist/docs/guides/bitbucket.html +62 -62
- package/dist/docs/guides/bitbucket.md +73 -134
- package/dist/docs/guides/cloud-agents.html +37 -37
- package/dist/docs/guides/cloud-agents.md +23 -53
- package/dist/docs/guides/convert-automation.html +39 -56
- package/dist/docs/guides/convert-automation.md +31 -115
- package/dist/docs/guides/github.html +46 -54
- package/dist/docs/guides/github.md +49 -131
- package/dist/docs/guides/gitlab.html +98 -108
- package/dist/docs/guides/gitlab.md +104 -184
- package/dist/docs/guides/improve.html +38 -43
- package/dist/docs/guides/improve.md +24 -90
- package/dist/docs/guides/jev.html +55 -95
- package/dist/docs/guides/jev.md +53 -160
- package/dist/docs/guides/mcp-oauth.html +38 -40
- package/dist/docs/guides/mcp-oauth.md +33 -67
- package/dist/docs/guides/migrate.html +38 -43
- package/dist/docs/guides/migrate.md +56 -130
- package/dist/docs/guides/opentelemetry.html +41 -43
- package/dist/docs/guides/opentelemetry.md +32 -27
- package/dist/docs/guides/pagerduty.html +50 -49
- package/dist/docs/guides/pagerduty.md +65 -199
- package/dist/docs/guides/slack.html +51 -57
- package/dist/docs/guides/slack.md +76 -204
- package/dist/docs/guides/webhooks.html +42 -42
- package/dist/docs/guides/webhooks.md +64 -146
- package/dist/docs/hashmap.json +1 -1
- package/dist/docs/hillclimbing.html +40 -42
- package/dist/docs/hillclimbing.md +40 -102
- package/dist/docs/index.html +55 -61
- package/dist/docs/index.md +49 -83
- package/dist/docs/llms-full.txt +2262 -5721
- package/dist/docs/llms.txt +9 -14
- package/dist/docs/quickstart.html +95 -152
- package/dist/docs/quickstart.md +91 -180
- package/dist/docs/reference/access.html +36 -36
- package/dist/docs/reference/access.md +7 -12
- package/dist/docs/reference/agent-config.html +43 -56
- package/dist/docs/reference/agent-config.md +91 -174
- package/dist/docs/reference/artifacts.html +38 -39
- package/dist/docs/reference/artifacts.md +38 -62
- package/dist/docs/reference/channels.html +38 -39
- package/dist/docs/reference/channels.md +102 -151
- package/dist/docs/reference/cli.html +68 -84
- package/dist/docs/reference/cli.md +232 -697
- package/dist/docs/reference/connections.html +45 -57
- package/dist/docs/reference/connections.md +83 -164
- package/dist/docs/reference/evals.html +47 -54
- package/dist/docs/reference/evals.md +75 -141
- package/dist/docs/reference/extensions.html +41 -43
- package/dist/docs/reference/extensions.md +37 -70
- package/dist/docs/reference/hooks.html +40 -52
- package/dist/docs/reference/hooks.md +62 -133
- package/dist/docs/reference/http-api.html +38 -42
- package/dist/docs/reference/http-api.md +66 -118
- package/dist/docs/reference/instructions.html +39 -39
- package/dist/docs/reference/instructions.md +19 -29
- package/dist/docs/reference/limits.html +38 -38
- package/dist/docs/reference/limits.md +9 -21
- package/dist/docs/reference/memory.html +37 -38
- package/dist/docs/reference/memory.md +38 -75
- package/dist/docs/reference/playground.html +36 -36
- package/dist/docs/reference/playground.md +14 -22
- package/dist/docs/reference/project-layout.html +49 -48
- package/dist/docs/reference/project-layout.md +38 -84
- package/dist/docs/reference/schedules.html +46 -48
- package/dist/docs/reference/schedules.md +59 -90
- package/dist/docs/reference/sessions.html +37 -37
- package/dist/docs/reference/sessions.md +66 -113
- package/dist/docs/reference/skills.html +39 -39
- package/dist/docs/reference/skills.md +11 -31
- package/dist/docs/reference/subagents.html +37 -37
- package/dist/docs/reference/subagents.md +17 -31
- package/dist/docs/reference/tools.html +47 -64
- package/dist/docs/reference/tools.md +48 -130
- package/dist/docs/templates/agentic-owners.html +38 -38
- package/dist/docs/templates/agentic-owners.md +50 -74
- package/dist/docs/templates/reviews.html +69 -0
- package/dist/docs/templates/reviews.md +71 -0
- package/dist/docs/troubleshooting.html +38 -36
- package/dist/docs/troubleshooting.md +48 -101
- package/dist/internal/cli-deploy.js +1 -1
- package/dist/internal/cli-docs.js +1 -1
- package/dist/internal/cli-images.js +1 -1
- package/dist/internal/cli-skills.js +1 -1
- package/dist/internal/cursor/credentials.js +1 -1
- package/dist/internal/deploy-manifest.d.ts +1 -0
- package/dist/internal/deploy-manifest.d.ts.map +1 -1
- package/dist/internal/deploy-manifest.js +0 -1
- package/dist/internal/discovery/agent-config.d.ts +2 -6
- package/dist/internal/discovery/agent-config.d.ts.map +1 -1
- package/dist/internal/discovery/agent-config.js +8 -25
- package/dist/internal/discovery/agent.d.ts.map +1 -1
- package/dist/internal/discovery/agent.js +2 -3
- package/dist/internal/discovery/project.d.ts.map +1 -1
- package/dist/internal/discovery/project.js +15 -0
- package/dist/internal/init-project.d.ts.map +1 -1
- package/dist/internal/init-project.js +7 -5
- package/dist/internal/reminder-runner.js +1 -1
- package/dist/internal/session-engine.js +1 -1
- package/dist/internal/shallow-clone.js +1 -1
- package/dist/playground/assets/index-D6mVn9og.js +62 -0
- package/dist/playground/assets/index-DlALM6gr.css +1 -0
- package/dist/playground/index.html +2 -2
- package/dist/types.d.ts +4 -17
- package/dist/types.d.ts.map +1 -1
- package/docs/README.md +49 -83
- package/docs/building-with-agents.md +11 -104
- package/docs/changelog.md +6 -156
- package/docs/deployment.md +83 -262
- package/docs/evals.md +34 -130
- package/docs/guides/agent-to-agent.md +27 -64
- package/docs/guides/bitbucket.md +73 -134
- package/docs/guides/cloud-agents.md +23 -53
- package/docs/guides/convert-automation.md +31 -115
- package/docs/guides/github.md +49 -131
- package/docs/guides/gitlab.md +104 -184
- package/docs/guides/improve.md +24 -90
- package/docs/guides/jev.md +53 -160
- package/docs/guides/mcp-oauth.md +33 -67
- package/docs/guides/migrate.md +56 -130
- package/docs/guides/opentelemetry.md +37 -27
- package/docs/guides/pagerduty.md +68 -200
- package/docs/guides/slack.md +76 -204
- package/docs/guides/webhooks.md +64 -146
- package/docs/hillclimbing.md +41 -103
- package/docs/quickstart.md +91 -180
- package/docs/reference/access.md +7 -12
- package/docs/reference/agent-config.md +91 -174
- package/docs/reference/artifacts.md +38 -62
- package/docs/reference/channels.md +102 -151
- package/docs/reference/cli.md +232 -697
- package/docs/reference/connections.md +83 -164
- package/docs/reference/evals.md +75 -141
- package/docs/reference/extensions.md +37 -70
- package/docs/reference/hooks.md +61 -132
- package/docs/reference/http-api.md +66 -118
- package/docs/reference/instructions.md +20 -30
- package/docs/reference/limits.md +10 -22
- package/docs/reference/memory.md +38 -75
- package/docs/reference/playground.md +14 -22
- package/docs/reference/project-layout.md +38 -84
- package/docs/reference/schedules.md +59 -90
- package/docs/reference/sessions.md +66 -113
- package/docs/reference/skills.md +12 -32
- package/docs/reference/subagents.md +17 -31
- package/docs/reference/tools.md +48 -130
- package/docs/templates/agentic-owners.md +50 -74
- package/docs/templates/reviews.md +76 -0
- package/docs/troubleshooting.md +48 -101
- package/package.json +2 -2
- package/skills/deploy/SKILL.md +1 -1
- package/skills/migrate-v2/SKILL.md +0 -1
- package/dist/docs/assets/building-with-agents.md.Bf1yH2tG.js +0 -9
- package/dist/docs/assets/changelog.md.DrLFQUlu.js +0 -1
- package/dist/docs/assets/changelog.md.DrLFQUlu.lean.js +0 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.Dpd_wxVU.js +0 -1
- package/dist/docs/assets/chunks/channel._fZX_hvU.js +0 -1
- package/dist/docs/assets/chunks/classDiagram-6PBFFD2Q.eClS1Q0E.js +0 -1
- package/dist/docs/assets/chunks/classDiagram-v2-HSJHXN6E.eClS1Q0E.js +0 -1
- package/dist/docs/assets/chunks/clone.W1Phdw9P.js +0 -1
- package/dist/docs/assets/chunks/stateDiagram-v2-QKLJ7IA2.77yf4kRr.js +0 -1
- package/dist/docs/assets/deployment.md.KW0Ni_rY.js +0 -31
- package/dist/docs/assets/evals.md.DiljU__Z.js +0 -68
- package/dist/docs/assets/evals.md.DiljU__Z.lean.js +0 -1
- package/dist/docs/assets/guides_agent-to-agent.md.D1D5f6Jc.lean.js +0 -1
- package/dist/docs/assets/guides_cloud-agents.md.Ceyjv0ul.js +0 -9
- package/dist/docs/assets/guides_convert-automation.md.BjraCvok.js +0 -43
- package/dist/docs/assets/guides_hooks.md.BjeEiM7s.js +0 -54
- package/dist/docs/assets/guides_hooks.md.BjeEiM7s.lean.js +0 -1
- package/dist/docs/assets/guides_improve.md.BFYtrtLE.js +0 -21
- package/dist/docs/assets/guides_migrate.md.RS9Eu-Xh.js +0 -8
- package/dist/docs/assets/guides_opentelemetry.md.CoZ1kNyd.js +0 -28
- package/dist/docs/assets/guides_opentelemetry.md.CoZ1kNyd.lean.js +0 -1
- package/dist/docs/assets/guides_pagerduty.md.mnKXe9U0.js +0 -60
- package/dist/docs/assets/guides_pagerduty.md.mnKXe9U0.lean.js +0 -1
- package/dist/docs/assets/guides_slack.md.BwXHURvY.js +0 -88
- package/dist/docs/assets/hillclimbing.md.By61f3uq.js +0 -7
- package/dist/docs/assets/hillclimbing.md.By61f3uq.lean.js +0 -1
- package/dist/docs/assets/index.md.CAruw_rp.js +0 -26
- package/dist/docs/assets/quickstart.md.CljDaJBi.js +0 -170
- package/dist/docs/assets/reference_access.md.CEtibb4i.js +0 -1
- package/dist/docs/assets/reference_access.md.CEtibb4i.lean.js +0 -1
- package/dist/docs/assets/reference_agent-config.md.Ck29qYNj.js +0 -40
- package/dist/docs/assets/reference_artifacts.md.C-hKR2hp.js +0 -18
- package/dist/docs/assets/reference_channels.md.DsJfGSBJ.js +0 -22
- package/dist/docs/assets/reference_cli.md.BkdLRbXg.js +0 -99
- package/dist/docs/assets/reference_cli.md.BkdLRbXg.lean.js +0 -1
- package/dist/docs/assets/reference_connections.md.BoAotr7_.js +0 -66
- package/dist/docs/assets/reference_evals.md.2BXeMiO2.js +0 -76
- package/dist/docs/assets/reference_hooks.md.B-bwJuiG.js +0 -51
- package/dist/docs/assets/reference_hooks.md.B-bwJuiG.lean.js +0 -1
- package/dist/docs/assets/reference_http-api.md._RWELr3d.js +0 -11
- package/dist/docs/assets/reference_instructions.md.yL2hXLnl.js +0 -14
- package/dist/docs/assets/reference_instructions.md.yL2hXLnl.lean.js +0 -1
- package/dist/docs/assets/reference_limits.md.BCCnvt3o.js +0 -8
- package/dist/docs/assets/reference_limits.md.BCCnvt3o.lean.js +0 -1
- package/dist/docs/assets/reference_memory.md.Dd7cyV9V.js +0 -4
- package/dist/docs/assets/reference_playground.md.CQiJk9CM.js +0 -1
- package/dist/docs/assets/reference_project-layout.md.BamcPDUn.js +0 -25
- package/dist/docs/assets/reference_project-layout.md.BamcPDUn.lean.js +0 -1
- package/dist/docs/assets/reference_prompt.md.BMjm1Jp3.js +0 -9
- package/dist/docs/assets/reference_prompt.md.BMjm1Jp3.lean.js +0 -1
- package/dist/docs/assets/reference_schedules.md.RC47aFm4.js +0 -47
- package/dist/docs/assets/reference_sessions.md.C7KFhWXV.js +0 -22
- package/dist/docs/assets/reference_subagents.md.CAhvW4yS.js +0 -10
- package/dist/docs/assets/reference_tools.md.DQr7Fnhu.js +0 -79
- package/dist/docs/assets/templates_agentic-owners.md.Dh2nDSqg.js +0 -13
- package/dist/docs/assets/templates_security-reviewer.md.DnwXdnB5.js +0 -3
- package/dist/docs/assets/templates_security-reviewer.md.DnwXdnB5.lean.js +0 -1
- package/dist/docs/assets/templates_thermo-quality-review.md.MJIdHkzE.js +0 -3
- package/dist/docs/assets/templates_thermo-quality-review.md.MJIdHkzE.lean.js +0 -1
- package/dist/docs/assets/templates_thermo-review.md.DfN1OFOl.js +0 -3
- package/dist/docs/assets/templates_thermo-review.md.DfN1OFOl.lean.js +0 -1
- package/dist/docs/assets/templates_triage.md.BYWMloV4.js +0 -9
- package/dist/docs/assets/templates_triage.md.BYWMloV4.lean.js +0 -1
- package/dist/docs/assets/troubleshooting.md.CGOdNVAL.js +0 -1
- package/dist/docs/guides/hooks.html +0 -113
- package/dist/docs/guides/hooks.md +0 -116
- package/dist/docs/package/skills/create-agent/SKILL.md +0 -178
- package/dist/docs/package/skills/evals/SKILL.md +0 -145
- package/dist/docs/package/skills/github/SKILL.md +0 -133
- package/dist/docs/package/skills/hillclimb/SKILL.md +0 -63
- package/dist/docs/package/skills/mcp-auth/SKILL.md +0 -60
- package/dist/docs/package/skills/migrate-v2/SKILL.md +0 -394
- package/dist/docs/package/skills/setup-slack/SKILL.md +0 -136
- package/dist/docs/reference/prompt.html +0 -68
- package/dist/docs/reference/prompt.md +0 -47
- package/dist/docs/templates/security-reviewer.html +0 -62
- package/dist/docs/templates/security-reviewer.md +0 -81
- package/dist/docs/templates/thermo-quality-review.html +0 -62
- package/dist/docs/templates/thermo-quality-review.md +0 -75
- package/dist/docs/templates/thermo-review.html +0 -62
- package/dist/docs/templates/thermo-review.md +0 -75
- package/dist/docs/templates/triage.html +0 -68
- package/dist/docs/templates/triage.md +0 -101
- package/dist/playground/assets/index-Cl9zvmmf.js +0 -62
- package/dist/playground/assets/index-F8b5WvWi.css +0 -1
- package/docs/guides/hooks.md +0 -121
- package/docs/reference/prompt.md +0 -52
- package/docs/templates/security-reviewer.md +0 -86
- package/docs/templates/thermo-quality-review.md +0 -80
- package/docs/templates/thermo-review.md +0 -80
- package/docs/templates/triage.md +0 -106
|
@@ -1,23 +1,19 @@
|
|
|
1
1
|
# Evals
|
|
2
2
|
|
|
3
3
|
Evals run fixed cases against an agent and record whether its turns,
|
|
4
|
-
tools, events, and output meet a contract.
|
|
5
|
-
|
|
6
|
-
discovers their IDs, executes them, and reports every assertion. See the
|
|
7
|
-
[Evals guide](/docs/evals.md) for the regression workflow.
|
|
4
|
+
tools, events, and output meet a contract. The
|
|
5
|
+
[Evals guide](/docs/evals.md) covers the regression workflow.
|
|
8
6
|
|
|
9
7
|
## Eval discovery and case IDs
|
|
10
8
|
|
|
11
9
|
Eval files live under the project-root `evals/` directory and end in
|
|
12
10
|
`.eval.ts` or `.eval.js`. The path under that directory becomes the
|
|
13
|
-
eval ID
|
|
11
|
+
eval ID, so `evals/readiness.eval.ts` becomes `readiness`,
|
|
12
|
+
`evals/builds/api.eval.ts` becomes `builds/api`, and
|
|
13
|
+
`evals/builds/index.eval.ts` becomes `builds`.
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
- `evals/builds/index.eval.ts` becomes `builds`.
|
|
18
|
-
|
|
19
|
-
Export one `defineEval` with either `test` or `cases`, not both. A case
|
|
20
|
-
ID appends one path segment to the file ID:
|
|
15
|
+
Export one `defineEval` that sets exactly one of `test` or `cases`. A
|
|
16
|
+
case ID adds one path segment to the file ID:
|
|
21
17
|
|
|
22
18
|
```ts
|
|
23
19
|
export default defineEval({
|
|
@@ -44,101 +40,87 @@ export default defineEval({
|
|
|
44
40
|
Those cases are `prs/checkout` and `prs/search` when the file is
|
|
45
41
|
`evals/prs.eval.ts`. Case IDs must be unique single path segments.
|
|
46
42
|
|
|
47
|
-
A file can also export an array of `defineEval` calls
|
|
48
|
-
|
|
49
|
-
handwritten scenarios stable IDs; arrays fit loaded datasets.
|
|
43
|
+
A file can also export an array of `defineEval` calls, named with
|
|
44
|
+
zero-padded indexes such as `sql/0000`.
|
|
50
45
|
|
|
51
|
-
`iterations` repeats one datapoint
|
|
52
|
-
`iterations: 3`, `weather/nyc`
|
|
53
|
-
`weather/nyc/
|
|
54
|
-
three.
|
|
46
|
+
`iterations` repeats one datapoint 1 to 100 times. With
|
|
47
|
+
`iterations: 3`, `weather/nyc` runs as `weather/nyc/1` through
|
|
48
|
+
`weather/nyc/3`, and selecting `weather/nyc` runs all three.
|
|
55
49
|
|
|
56
50
|
## Eval configuration
|
|
57
51
|
|
|
58
|
-
Every
|
|
59
|
-
`maxConcurrency`:
|
|
52
|
+
Every suite needs `evals/evals.config.ts` with `maxConcurrency`:
|
|
60
53
|
|
|
61
54
|
```ts
|
|
62
55
|
import { defineEvalConfig } from "@grok/agent-sdk/evals";
|
|
63
56
|
|
|
64
57
|
export default defineEvalConfig({
|
|
65
58
|
maxConcurrency: 20,
|
|
66
|
-
timeoutMs: 180_000,
|
|
67
59
|
judge: { model: "gpt-5.4-mini" },
|
|
68
60
|
});
|
|
69
61
|
```
|
|
70
62
|
|
|
71
63
|
| Option | Contract |
|
|
72
64
|
| --- | --- |
|
|
73
|
-
| `maxConcurrency` | Required
|
|
74
|
-
| `timeoutMs` | Per-case timeout
|
|
65
|
+
| `maxConcurrency` | Required. Concurrent authored datapoints, from 1 to 200 |
|
|
66
|
+
| `timeoutMs` | Per-case timeout. A case or file value wins, then the CLI flag, then this config |
|
|
75
67
|
| `judge` | Default model for `t.judge` |
|
|
76
68
|
| `reporters` | Objects notified as cases and runs complete |
|
|
77
|
-
| `maxPlaygroundRuns` |
|
|
69
|
+
| `maxPlaygroundRuns` | Server-side batches kept in playground history. Defaults to 20 |
|
|
78
70
|
|
|
79
71
|
A case can override `description`, `tags`, `timeoutMs`, `iterations`,
|
|
80
|
-
`judge`, `reporters`, and `metadata`. Case metadata merges over
|
|
81
|
-
|
|
72
|
+
`judge`, `reporters`, and `metadata`. Case metadata merges over file
|
|
73
|
+
metadata, and case reporters add to the file's reporters.
|
|
82
74
|
|
|
83
75
|
## Send turns in a case
|
|
84
76
|
|
|
85
77
|
`await t.send(message, options?)` runs one turn and waits until it
|
|
86
|
-
finishes or fails.
|
|
87
|
-
|
|
88
|
-
The returned turn exposes its assistant `message`, `sessionId`,
|
|
89
|
-
`events`, ordered `toolCalls`, `ok`, and one-based `index`. Assertions
|
|
90
|
-
on the turn inspect only that turn; assertions on `t` inspect the whole
|
|
91
|
-
case. Run values include `t.reply`, `t.events`, `t.turns`,
|
|
92
|
-
`t.sessionId`, `t.iteration`, `t.iterations`, and the timeout
|
|
93
|
-
`t.signal`.
|
|
78
|
+
finishes or fails. Sends in one test share a session.
|
|
94
79
|
|
|
95
|
-
|
|
80
|
+
The returned turn has the assistant `message`, `sessionId`, `events`,
|
|
81
|
+
ordered `toolCalls`, `ok`, and a one-based `index`. Assertions on a turn
|
|
82
|
+
inspect only that turn, and assertions on `t` inspect the whole case.
|
|
83
|
+
The case also exposes `t.reply`, `t.events`, `t.turns`, `t.sessionId`,
|
|
84
|
+
`t.iteration`, `t.iterations`, and the timeout `t.signal`.
|
|
96
85
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
| `workspaceDir` | Absolute working directory for a new local session |
|
|
86
|
+
On the first send only, `workspaceFiles` seeds the session workspace
|
|
87
|
+
from a map of relative paths to content, and `workspaceDir` sets an
|
|
88
|
+
absolute working directory for a new local session.
|
|
101
89
|
|
|
102
|
-
|
|
103
|
-
|
|
90
|
+
`turn.expectOk()` throws when the turn failed, so later steps that
|
|
91
|
+
depend on it don't run.
|
|
104
92
|
|
|
105
93
|
## Trajectory assertions
|
|
106
94
|
|
|
107
|
-
Assertions record failures without stopping the test
|
|
108
|
-
reports every violated contract.
|
|
95
|
+
Assertions record failures without stopping the test.
|
|
109
96
|
|
|
110
97
|
| Assertion | Checks |
|
|
111
98
|
| --- | --- |
|
|
112
99
|
| `t.succeeded()` | Run finished without failure |
|
|
113
|
-
| `t.calledTool(name, matcher?)` |
|
|
114
|
-
| `t.notCalledTool(name)` | No request for that tool occurred |
|
|
100
|
+
| `t.calledTool(name, matcher?)`, `t.notCalledTool(name)` | A matching tool request occurred, or none did |
|
|
115
101
|
| `t.loadedSkill(name)` | Agent opened a named skill |
|
|
116
|
-
| `t.toolOrder(names)` | Tool requests appeared in relative order |
|
|
117
|
-
| `t.usedNoTools()` | No tool was requested |
|
|
118
|
-
| `t.maxToolCalls(max)` | Tool-call count stayed under the bound |
|
|
102
|
+
| `t.toolOrder(names)`, `t.eventOrder(matchers)` | Tool requests or event groups appeared in relative order |
|
|
103
|
+
| `t.usedNoTools()`, `t.maxToolCalls(max)` | No tool was requested, or the count stayed under the bound |
|
|
119
104
|
| `t.noFailedActions()` | No tool result failed |
|
|
120
105
|
| `t.calledSubagent(name, matcher?)` | Matching subagent call occurred |
|
|
121
106
|
| `t.taggedArtifact(kind?, predicate?)` | Matching durable artifact was tagged |
|
|
122
|
-
| `t.event(type, matcher?)` |
|
|
123
|
-
| `t.notEvent(type, matcher?)` | Matching session event did not occur |
|
|
124
|
-
| `t.eventOrder(matchers)` | Event groups appeared in relative order |
|
|
107
|
+
| `t.event(type, matcher?)`, `t.notEvent(type, matcher?)` | A matching session event occurred, or none did |
|
|
125
108
|
| `t.eventsSatisfy(label, predicate)` | Predicate accepted the event stream |
|
|
126
109
|
| `t.check(value, expectation)` | Value met an expectation builder |
|
|
127
110
|
| `t.score(name, value)` | Recorded a numeric score from 0 to 1 |
|
|
128
111
|
|
|
129
112
|
`t.requireToolCall`, `t.requireInputRequest`, and `t.require` return the
|
|
130
|
-
matched value and stop the
|
|
113
|
+
matched value and stop the rest of the test when nothing matches.
|
|
131
114
|
|
|
132
115
|
## Tool and event matchers
|
|
133
116
|
|
|
134
117
|
Tool matchers can narrow by `input`, `output`, `status`, and `count`.
|
|
135
|
-
Values
|
|
136
|
-
literals
|
|
118
|
+
Values can be literals, regular expressions, or predicates. Object
|
|
119
|
+
literals match partially:
|
|
137
120
|
|
|
138
121
|
```ts
|
|
139
122
|
t.calledTool("inspect_pr", {
|
|
140
123
|
input: { verdict: "review" },
|
|
141
|
-
status: "completed",
|
|
142
124
|
count: 1,
|
|
143
125
|
});
|
|
144
126
|
```
|
|
@@ -149,43 +131,34 @@ Event matchers accept `data` and `count`.
|
|
|
149
131
|
|
|
150
132
|
## Expectation builders
|
|
151
133
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
| `includes(string | RegExp)` | Hard gate |
|
|
157
|
-
| `equals(value)` | Hard gate |
|
|
158
|
-
| `matches(schema)` | Hard gate |
|
|
159
|
-
| `satisfies(predicate, label)` | Hard gate |
|
|
160
|
-
| `similarity(expected)` | Soft score |
|
|
161
|
-
|
|
162
|
-
`matches` accepts Standard Schema implementations such as Zod.
|
|
134
|
+
`includes(string | RegExp)`, `equals(value)`, `matches(schema)`, and
|
|
135
|
+
`satisfies(predicate, label)` are hard gates by default, and
|
|
136
|
+
`similarity(expected)` is a soft score. `matches` accepts Standard
|
|
137
|
+
Schema implementations such as Zod.
|
|
163
138
|
`normalizedSimilarity(value, expected)` returns the same 0-to-1 value
|
|
164
|
-
|
|
139
|
+
that `similarity` uses.
|
|
165
140
|
|
|
166
141
|
## Severity and verdicts
|
|
167
142
|
|
|
168
143
|
Every assertion returns a severity handle:
|
|
169
144
|
|
|
170
145
|
```ts
|
|
171
|
-
t.succeeded();
|
|
172
146
|
t.calledTool("search").soft();
|
|
173
147
|
t.check(t.reply, similarity("Expected answer")).atLeast(0.8);
|
|
174
148
|
t.judge.closedQA("names the failing step").gate(1);
|
|
175
149
|
```
|
|
176
150
|
|
|
177
|
-
- `.gate(threshold?)` fails the case
|
|
178
|
-
- `.soft(threshold?)` records a value
|
|
179
|
-
- `.atLeast(threshold)`
|
|
180
|
-
verdict.
|
|
151
|
+
- `.gate(threshold?)` fails the case on a miss.
|
|
152
|
+
- `.soft(threshold?)` records a value and never fails the case.
|
|
153
|
+
- `.atLeast(threshold)` sets a soft bar, and a miss gives the case a
|
|
154
|
+
`scored` verdict.
|
|
181
155
|
|
|
182
|
-
Case verdicts are `passed`, `failed`, `scored`, or `skipped`.
|
|
183
|
-
|
|
184
|
-
set. Exit 2 means the selection matched no cases.
|
|
156
|
+
Case verdicts are `passed`, `failed`, `scored`, or `skipped`. A `scored`
|
|
157
|
+
case fails the run only under `--strict`.
|
|
185
158
|
|
|
186
159
|
## Judges
|
|
187
160
|
|
|
188
|
-
|
|
161
|
+
Judges grade what deterministic checks can't express:
|
|
189
162
|
|
|
190
163
|
```ts
|
|
191
164
|
t.judge.factuality(expected).atLeast(0.8);
|
|
@@ -194,26 +167,20 @@ t.judge.closedQA("names the root cause").gate(1);
|
|
|
194
167
|
t.judge.sql(expected).atLeast(0.9);
|
|
195
168
|
```
|
|
196
169
|
|
|
197
|
-
Each grades `t.reply`
|
|
198
|
-
|
|
199
|
-
call,
|
|
170
|
+
Each judge grades `t.reply` unless you pass `{ on }` with another value.
|
|
171
|
+
Set judge options in the project config, the eval, the case, or the
|
|
172
|
+
call, and the nearest value wins.
|
|
200
173
|
|
|
201
174
|
`t.judge.model(prompt)` returns a raw model reply for custom grading.
|
|
202
|
-
|
|
175
|
+
Agent output is untrusted input, so wrap it with `fenceUntrusted` and
|
|
203
176
|
include `EVAL_JUDGE_INJECTION_GUARD` in the judge prompt.
|
|
204
177
|
|
|
205
178
|
## Judge host
|
|
206
179
|
|
|
207
|
-
A judge
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
same case grades on macOS, a Linux cloud agent, and a Linux CI runner.
|
|
212
|
-
|
|
213
|
-
Without a credential, or when that credential belongs to a different
|
|
214
|
-
backend than the SDK will call, the case is skipped and the suite
|
|
215
|
-
continues. Without a judge model, the case fails and the assertion names
|
|
216
|
-
where to set one.
|
|
180
|
+
A judge needs a judge model and a Cursor credential from
|
|
181
|
+
`agent-sdk login`, `CURSOR_API_KEY`, or `CURSOR_SERVICE_ACCOUNT_KEY`.
|
|
182
|
+
Without a usable credential the runner skips the case, and without a
|
|
183
|
+
judge model it fails the case and names where to set one.
|
|
217
184
|
|
|
218
185
|
## Datasets and fixtures
|
|
219
186
|
|
|
@@ -223,16 +190,14 @@ Load committed JSON, JSONL, or YAML relative to the project root:
|
|
|
223
190
|
import { defineEval, includes } from "@grok/agent-sdk/evals";
|
|
224
191
|
import { loadYaml } from "@grok/agent-sdk/evals/loaders";
|
|
225
192
|
|
|
226
|
-
const rows = await loadYaml<
|
|
227
|
-
|
|
228
|
-
|
|
193
|
+
const rows = await loadYaml<Array<{ prompt: string; expected: string }>>(
|
|
194
|
+
"evals/data/cases.yaml",
|
|
195
|
+
);
|
|
229
196
|
|
|
230
197
|
export default rows.map(row =>
|
|
231
198
|
defineEval({
|
|
232
|
-
description: row.task,
|
|
233
199
|
async test(t) {
|
|
234
200
|
await t.send(row.prompt);
|
|
235
|
-
t.succeeded();
|
|
236
201
|
t.check(t.reply, includes(row.expected));
|
|
237
202
|
},
|
|
238
203
|
})
|
|
@@ -242,13 +207,18 @@ export default rows.map(row =>
|
|
|
242
207
|
## Reporters and results
|
|
243
208
|
|
|
244
209
|
Built-in reporters include `JUnit({ filePath, suiteName? })` and
|
|
245
|
-
`Artifacts({ dir })`.
|
|
210
|
+
`Artifacts({ dir })`. A custom reporter can implement `onRunStart`,
|
|
246
211
|
`onEvalComplete`, and `onRunComplete`.
|
|
247
212
|
|
|
248
|
-
JSON output
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
213
|
+
JSON output has run totals and one result per case. Each result has the
|
|
214
|
+
case ID, verdict, assertions, session ID, inputs, final text, tool
|
|
215
|
+
calls, metrics, logs, duration, metadata, tags, and any error or skip
|
|
216
|
+
reason.
|
|
217
|
+
|
|
218
|
+
Each local run writes `summary.json`, `results.jsonl`, and
|
|
219
|
+
`evals/<case-id>.json` under `evals/<stamp>/` in the project state
|
|
220
|
+
directory. `--state-root` doesn't move them, and `--artifacts <dir>` or
|
|
221
|
+
`--no-artifacts` does.
|
|
252
222
|
|
|
253
223
|
## Select cases
|
|
254
224
|
|
|
@@ -259,17 +229,14 @@ agent-sdk eval --dir . builds/checkout
|
|
|
259
229
|
agent-sdk eval --dir . --tag smoke
|
|
260
230
|
```
|
|
261
231
|
|
|
262
|
-
ID
|
|
263
|
-
|
|
264
|
-
|
|
232
|
+
An ID matches itself and its descendants. Repeated IDs or tags combine
|
|
233
|
+
with OR, and a case must match both groups when you pass both.
|
|
234
|
+
[Run evals](/docs/reference/cli.md#eval) covers runs on a server.
|
|
265
235
|
|
|
266
236
|
## Suite summary cases
|
|
267
237
|
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
aggregator by id, or with `--tag summary`, also runs every other case
|
|
271
|
-
from that file. A selection that includes no `summary` case stays as
|
|
272
|
-
you wrote it. `--list` shows the same set a run would execute.
|
|
238
|
+
A case tagged `EVAL_SUMMARY_TAG` aggregates its file, so selecting it by
|
|
239
|
+
ID or with `--tag summary` also runs every other case in that file.
|
|
273
240
|
|
|
274
241
|
```ts
|
|
275
242
|
import { defineEval, EVAL_SUMMARY_TAG } from "@grok/agent-sdk/evals";
|
|
@@ -294,36 +261,3 @@ export default defineEval({
|
|
|
294
261
|
],
|
|
295
262
|
});
|
|
296
263
|
```
|
|
297
|
-
|
|
298
|
-
When that file is `evals/prs.eval.ts`, `agent-sdk eval --dir . prs/overall`
|
|
299
|
-
runs `prs/checkout` and `prs/overall`.
|
|
300
|
-
|
|
301
|
-
## Inspect run artifacts
|
|
302
|
-
|
|
303
|
-
Default run artifacts go under `evals/<stamp>/` in the project state
|
|
304
|
-
directory and include
|
|
305
|
-
`summary.json`, `results.jsonl`, and `evals/<case-id>.json`.
|
|
306
|
-
`--artifacts <dir>` chooses another destination; `--no-artifacts`
|
|
307
|
-
disables them. This location is independent of `--state-root`.
|
|
308
|
-
|
|
309
|
-
## Run evals against a server
|
|
310
|
-
|
|
311
|
-
The playground **Evals** view starts batches on its running server, and
|
|
312
|
-
`--url` targets a named running server:
|
|
313
|
-
|
|
314
|
-
```text
|
|
315
|
-
agent-sdk eval --url http://127.0.0.1:3000/vulnerability-scanner --tag smoke
|
|
316
|
-
agent-sdk eval get <eval-id> --url http://127.0.0.1:3000/vulnerability-scanner
|
|
317
|
-
agent-sdk eval cancel <eval-id> --url http://127.0.0.1:3000/vulnerability-scanner
|
|
318
|
-
```
|
|
319
|
-
|
|
320
|
-
Server-side batches use the target's discovered cases and return an eval
|
|
321
|
-
ID for `status` or `cancel`. Local reporter and concurrency flags don't
|
|
322
|
-
apply to `--url` runs.
|
|
323
|
-
|
|
324
|
-
## Related
|
|
325
|
-
|
|
326
|
-
- [Evals guide](/docs/evals.md)
|
|
327
|
-
- [CLI reference](/docs/reference/cli.md#eval)
|
|
328
|
-
- [Sessions](/docs/reference/sessions.md)
|
|
329
|
-
- [Artifacts](/docs/reference/artifacts.md)
|