@grok/agent-sdk 0.2.29 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channels/slack/cli-project.js +1 -1
- package/dist/docs/404.html +2 -2
- package/dist/docs/assets/{app.DMrZKFjI.js → app.BzTUBOAD.js} +4 -4
- package/dist/docs/assets/building-with-agents.md.BqJ7AAjG.js +6 -0
- package/dist/docs/assets/{building-with-agents.md.Bf1yH2tG.lean.js → building-with-agents.md.BqJ7AAjG.lean.js} +1 -1
- package/dist/docs/assets/changelog.md.BWhC6DET.js +1 -0
- package/dist/docs/assets/changelog.md.BWhC6DET.lean.js +1 -0
- package/dist/docs/assets/chunks/@localSearchIndexroot.CWxwVoCG.js +1 -0
- package/dist/docs/assets/chunks/{VPLocalSearchBox.DQvuw1KA.js → VPLocalSearchBox.DsnRlUT5.js} +1 -1
- package/dist/docs/assets/chunks/{arc.ahOG-Qza.js → arc.DH8GOhiv.js} +1 -1
- package/dist/docs/assets/chunks/{architectureDiagram-Q4EWVU46.BcgfYNO_.js → architectureDiagram-Q4EWVU46.9m9ObyuE.js} +1 -1
- package/dist/docs/assets/chunks/{baseUniq.DeIYZrWh.js → baseUniq.BXHpx1lE.js} +1 -1
- package/dist/docs/assets/chunks/{blockDiagram-DXYQGD6D.HjJ2goMX.js → blockDiagram-DXYQGD6D.xY3EX59q.js} +1 -1
- package/dist/docs/assets/chunks/{c4Diagram-AHTNJAMY.C1QTb9GM.js → c4Diagram-AHTNJAMY.DnqgMVgc.js} +1 -1
- package/dist/docs/assets/chunks/channel.Cctv0f7X.js +1 -0
- package/dist/docs/assets/chunks/{chunk-4BX2VUAB.UuZiDiVp.js → chunk-4BX2VUAB.BjFxK2A3.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-4TB4RGXK.5A3YmkhI.js → chunk-4TB4RGXK.BujS_Spt.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-55IACEB6.B4uf1yYV.js → chunk-55IACEB6.YbTjhdaM.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-EDXVE4YY.CoXBce8t.js → chunk-EDXVE4YY.DnQ8t4GC.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-FMBD7UC4.DnC5NO4e.js → chunk-FMBD7UC4.B67AwAmj.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-OYMX7WX6.CzfHCeIi.js → chunk-OYMX7WX6.DrZIAFOm.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-QZHKN3VN.C6tcwkVd.js → chunk-QZHKN3VN.BLmmydkn.js} +1 -1
- package/dist/docs/assets/chunks/{chunk-YZCP3GAM._oboyZ1A.js → chunk-YZCP3GAM.DaoOfw0O.js} +1 -1
- package/dist/docs/assets/chunks/classDiagram-6PBFFD2Q.DU5sqchW.js +1 -0
- package/dist/docs/assets/chunks/classDiagram-v2-HSJHXN6E.DU5sqchW.js +1 -0
- package/dist/docs/assets/chunks/clone.B9rdNt6j.js +1 -0
- package/dist/docs/assets/chunks/{cose-bilkent-S5V4N54A.BfPVKeId.js → cose-bilkent-S5V4N54A.DMQuNhM5.js} +1 -1
- package/dist/docs/assets/chunks/{dagre-KV5264BT.CLMLc0lD.js → dagre-KV5264BT.CZoGVuUf.js} +1 -1
- package/dist/docs/assets/chunks/{diagram-5BDNPKRD.C7OF7UfH.js → diagram-5BDNPKRD.CLt9uOyL.js} +1 -1
- package/dist/docs/assets/chunks/{diagram-G4DWMVQ6.Dpao0Bl1.js → diagram-G4DWMVQ6.DpczEm1G.js} +1 -1
- package/dist/docs/assets/chunks/{diagram-MMDJMWI5.DLnM9eLp.js → diagram-MMDJMWI5.D4r13DvJ.js} +1 -1
- package/dist/docs/assets/chunks/{diagram-TYMM5635.BcpVgA_M.js → diagram-TYMM5635.CYU9eGFu.js} +1 -1
- package/dist/docs/assets/chunks/{erDiagram-SMLLAGMA.CsL5JuCV.js → erDiagram-SMLLAGMA.QA9naWEw.js} +1 -1
- package/dist/docs/assets/chunks/{flowDiagram-DWJPFMVM.MFihq9AF.js → flowDiagram-DWJPFMVM.BbNoVZYd.js} +1 -1
- package/dist/docs/assets/chunks/{ganttDiagram-T4ZO3ILL.CJeLodyG.js → ganttDiagram-T4ZO3ILL.IHjDoG7r.js} +1 -1
- package/dist/docs/assets/chunks/{gitGraphDiagram-UUTBAWPF.1HhlE45Z.js → gitGraphDiagram-UUTBAWPF.B21VMdf5.js} +1 -1
- package/dist/docs/assets/chunks/{graph.BS02KkWA.js → graph.DgJ4L-8V.js} +1 -1
- package/dist/docs/assets/chunks/{infoDiagram-42DDH7IO.fp5gvytQ.js → infoDiagram-42DDH7IO.CQjrcthc.js} +1 -1
- package/dist/docs/assets/chunks/{ishikawaDiagram-UXIWVN3A.C6Wk1V4I.js → ishikawaDiagram-UXIWVN3A.D4tPGtVo.js} +1 -1
- package/dist/docs/assets/chunks/{journeyDiagram-VCZTEJTY.qJt4uJ_H.js → journeyDiagram-VCZTEJTY.BD3WaNbQ.js} +1 -1
- package/dist/docs/assets/chunks/{kanban-definition-6JOO6SKY.z2tG9kly.js → kanban-definition-6JOO6SKY.Ca0o4Tqz.js} +1 -1
- package/dist/docs/assets/chunks/{layout.BPCkdejr.js → layout.DXXqX4MQ.js} +1 -1
- package/dist/docs/assets/chunks/{linear.CL3TC3u7.js → linear.5c61gNw4.js} +1 -1
- package/dist/docs/assets/chunks/{min.SPrSp704.js → min.B1-NAjHX.js} +1 -1
- package/dist/docs/assets/chunks/{mindmap-definition-QFDTVHPH.DgrQRFZW.js → mindmap-definition-QFDTVHPH.BELznv3-.js} +1 -1
- package/dist/docs/assets/chunks/{pieDiagram-DEJITSTG.Ba7yxiW2.js → pieDiagram-DEJITSTG.qPzO1mq5.js} +1 -1
- package/dist/docs/assets/chunks/{quadrantDiagram-34T5L4WZ.Bvskad4p.js → quadrantDiagram-34T5L4WZ.CxIvzbjX.js} +1 -1
- package/dist/docs/assets/chunks/{requirementDiagram-MS252O5E.Ce70ugV9.js → requirementDiagram-MS252O5E.CqA0iJx8.js} +1 -1
- package/dist/docs/assets/chunks/{sankeyDiagram-XADWPNL6.BJ5OEULO.js → sankeyDiagram-XADWPNL6.XRTutBnb.js} +1 -1
- package/dist/docs/assets/chunks/{sequenceDiagram-FGHM5R23.C8a1hDs6.js → sequenceDiagram-FGHM5R23.E--7bRLQ.js} +1 -1
- package/dist/docs/assets/chunks/{stateDiagram-FHFEXIEX.D2D8A5CK.js → stateDiagram-FHFEXIEX.C4vGu8zq.js} +1 -1
- package/dist/docs/assets/chunks/stateDiagram-v2-QKLJ7IA2.ECMu2BdJ.js +1 -0
- package/dist/docs/assets/chunks/{theme.DRciHmS1.js → theme.DMj_7q3B.js} +2 -2
- package/dist/docs/assets/chunks/{timeline-definition-GMOUNBTQ.C4ePrY8-.js → timeline-definition-GMOUNBTQ.C7LgNVAL.js} +1 -1
- package/dist/docs/assets/chunks/{vennDiagram-DHZGUBPP.Dzso6dEA.js → vennDiagram-DHZGUBPP.HUwoZcy_.js} +1 -1
- package/dist/docs/assets/chunks/{wardley-RL74JXVD.djgXR0ar.js → wardley-RL74JXVD.BxFEFYZS.js} +1 -1
- package/dist/docs/assets/chunks/{wardleyDiagram-NUSXRM2D.Bf2ICxjh.js → wardleyDiagram-NUSXRM2D.B2pSo5rB.js} +1 -1
- package/dist/docs/assets/chunks/{xychartDiagram-5P7HB3ND.DEH4BcVJ.js → xychartDiagram-5P7HB3ND.NQhMiOOL.js} +1 -1
- package/dist/docs/assets/deployment.md.BtqJrHhp.js +31 -0
- package/dist/docs/assets/{deployment.md.KW0Ni_rY.lean.js → deployment.md.BtqJrHhp.lean.js} +1 -1
- package/dist/docs/assets/evals.md.Dne6VRmZ.js +49 -0
- package/dist/docs/assets/evals.md.Dne6VRmZ.lean.js +1 -0
- package/dist/docs/assets/{guides_agent-to-agent.md.D1D5f6Jc.js → guides_agent-to-agent.md.BcmSqL2z.js} +8 -8
- package/dist/docs/assets/guides_agent-to-agent.md.BcmSqL2z.lean.js +1 -0
- package/dist/docs/assets/{guides_bitbucket.md.DkjkBRU8.js → guides_bitbucket.md.BiA6ZMFW.js} +27 -27
- package/dist/docs/assets/{guides_bitbucket.md.DkjkBRU8.lean.js → guides_bitbucket.md.BiA6ZMFW.lean.js} +1 -1
- package/dist/docs/assets/guides_cloud-agents.md.CiXAW6on.js +9 -0
- package/dist/docs/assets/{guides_cloud-agents.md.Ceyjv0ul.lean.js → guides_cloud-agents.md.CiXAW6on.lean.js} +1 -1
- package/dist/docs/assets/guides_convert-automation.md.BvTxsJ8J.js +26 -0
- package/dist/docs/assets/{guides_convert-automation.md.BjraCvok.lean.js → guides_convert-automation.md.BvTxsJ8J.lean.js} +1 -1
- package/dist/docs/assets/{guides_github.md.CgN5sNbW.js → guides_github.md.6jiEH7dg.js} +11 -19
- package/dist/docs/assets/{guides_github.md.CgN5sNbW.lean.js → guides_github.md.6jiEH7dg.lean.js} +1 -1
- package/dist/docs/assets/{guides_gitlab.md.DCE0p7_w.js → guides_gitlab.md.DcKusOcU.js} +63 -73
- package/dist/docs/assets/{guides_gitlab.md.DCE0p7_w.lean.js → guides_gitlab.md.DcKusOcU.lean.js} +1 -1
- package/dist/docs/assets/guides_improve.md.o3WOGR7n.js +16 -0
- package/dist/docs/assets/{guides_improve.md.BFYtrtLE.lean.js → guides_improve.md.o3WOGR7n.lean.js} +1 -1
- package/dist/docs/assets/{guides_jev.md.CEqLgjj9.js → guides_jev.md.DGCzkMUN.js} +20 -60
- package/dist/docs/assets/{guides_jev.md.CEqLgjj9.lean.js → guides_jev.md.DGCzkMUN.lean.js} +1 -1
- package/dist/docs/assets/{guides_mcp-oauth.md.DrGY92u4.js → guides_mcp-oauth.md.C-Sx_IPa.js} +3 -5
- package/dist/docs/assets/{guides_mcp-oauth.md.DrGY92u4.lean.js → guides_mcp-oauth.md.C-Sx_IPa.lean.js} +1 -1
- package/dist/docs/assets/guides_migrate.md.BbhHYeCr.js +3 -0
- package/dist/docs/assets/{guides_migrate.md.RS9Eu-Xh.lean.js → guides_migrate.md.BbhHYeCr.lean.js} +1 -1
- package/dist/docs/assets/guides_opentelemetry.md.BRSktBHt.js +26 -0
- package/dist/docs/assets/guides_opentelemetry.md.BRSktBHt.lean.js +1 -0
- package/dist/docs/assets/guides_pagerduty.md.pMXUkZZc.js +61 -0
- package/dist/docs/assets/guides_pagerduty.md.pMXUkZZc.lean.js +1 -0
- package/dist/docs/assets/guides_slack.md.BgIGG9ur.js +82 -0
- package/dist/docs/assets/{guides_slack.md.BwXHURvY.lean.js → guides_slack.md.BgIGG9ur.lean.js} +1 -1
- package/dist/docs/assets/{guides_webhooks.md.BJk63ZTs.js → guides_webhooks.md.B5DXt2v1.js} +7 -7
- package/dist/docs/assets/{guides_webhooks.md.BJk63ZTs.lean.js → guides_webhooks.md.B5DXt2v1.lean.js} +1 -1
- package/dist/docs/assets/hillclimbing.md.3TkdF79A.js +5 -0
- package/dist/docs/assets/hillclimbing.md.3TkdF79A.lean.js +1 -0
- package/dist/docs/assets/index.md.DRfaoRCZ.js +20 -0
- package/dist/docs/assets/{index.md.CAruw_rp.lean.js → index.md.DRfaoRCZ.lean.js} +1 -1
- package/dist/docs/assets/quickstart.md.r-oE8nkX.js +113 -0
- package/dist/docs/assets/{quickstart.md.CljDaJBi.lean.js → quickstart.md.r-oE8nkX.lean.js} +1 -1
- package/dist/docs/assets/reference_access.md.C5lblfUC.js +1 -0
- package/dist/docs/assets/reference_access.md.C5lblfUC.lean.js +1 -0
- package/dist/docs/assets/reference_agent-config.md.C0Tp_Fk9.js +27 -0
- package/dist/docs/assets/{reference_agent-config.md.Ck29qYNj.lean.js → reference_agent-config.md.C0Tp_Fk9.lean.js} +1 -1
- package/dist/docs/assets/reference_artifacts.md.BJpAI7h9.js +17 -0
- package/dist/docs/assets/{reference_artifacts.md.C-hKR2hp.lean.js → reference_artifacts.md.BJpAI7h9.lean.js} +1 -1
- package/dist/docs/assets/reference_channels.md.CPWOpdsC.js +21 -0
- package/dist/docs/assets/{reference_channels.md.DsJfGSBJ.lean.js → reference_channels.md.CPWOpdsC.lean.js} +1 -1
- package/dist/docs/assets/reference_cli.md.B8GdpaRj.js +83 -0
- package/dist/docs/assets/reference_cli.md.B8GdpaRj.lean.js +1 -0
- package/dist/docs/assets/reference_connections.md.DoY4vrDH.js +54 -0
- package/dist/docs/assets/{reference_connections.md.BoAotr7_.lean.js → reference_connections.md.DoY4vrDH.lean.js} +1 -1
- package/dist/docs/assets/reference_evals.md.DgfUtfIc.js +69 -0
- package/dist/docs/assets/{reference_evals.md.2BXeMiO2.lean.js → reference_evals.md.DgfUtfIc.lean.js} +1 -1
- package/dist/docs/assets/{reference_extensions.md.DI5X5x8p.js → reference_extensions.md.DHbW6eOf.js} +6 -8
- package/dist/docs/assets/{reference_extensions.md.DI5X5x8p.lean.js → reference_extensions.md.DHbW6eOf.lean.js} +1 -1
- package/dist/docs/assets/reference_hooks.md.CSWmOwMM.js +39 -0
- package/dist/docs/assets/reference_hooks.md.CSWmOwMM.lean.js +1 -0
- package/dist/docs/assets/reference_http-api.md.C7Cb0r2J.js +7 -0
- package/dist/docs/assets/{reference_http-api.md._RWELr3d.lean.js → reference_http-api.md.C7Cb0r2J.lean.js} +1 -1
- package/dist/docs/assets/reference_instructions.md.CV6Qq0UX.js +14 -0
- package/dist/docs/assets/reference_instructions.md.CV6Qq0UX.lean.js +1 -0
- package/dist/docs/assets/reference_limits.md.CbdtFr4T.js +8 -0
- package/dist/docs/assets/reference_limits.md.CbdtFr4T.lean.js +1 -0
- package/dist/docs/assets/reference_memory.md.CdoJISM-.js +3 -0
- package/dist/docs/assets/{reference_memory.md.Dd7cyV9V.lean.js → reference_memory.md.CdoJISM-.lean.js} +1 -1
- package/dist/docs/assets/reference_playground.md.DIUsRAcy.js +1 -0
- package/dist/docs/assets/{reference_playground.md.CQiJk9CM.lean.js → reference_playground.md.DIUsRAcy.lean.js} +1 -1
- package/dist/docs/assets/reference_project-layout.md.9DiKoE16.js +26 -0
- package/dist/docs/assets/reference_project-layout.md.9DiKoE16.lean.js +1 -0
- package/dist/docs/assets/reference_schedules.md.RhQ9ZLo1.js +45 -0
- package/dist/docs/assets/{reference_schedules.md.RC47aFm4.lean.js → reference_schedules.md.RhQ9ZLo1.lean.js} +1 -1
- package/dist/docs/assets/reference_sessions.md.NXVQt9mP.js +22 -0
- package/dist/docs/assets/{reference_sessions.md.C7KFhWXV.lean.js → reference_sessions.md.NXVQt9mP.lean.js} +1 -1
- package/dist/docs/assets/{reference_skills.md.Dej2zQxi.js → reference_skills.md.DzCPNPkG.js} +4 -4
- package/dist/docs/assets/{reference_skills.md.Dej2zQxi.lean.js → reference_skills.md.DzCPNPkG.lean.js} +1 -1
- package/dist/docs/assets/reference_subagents.md.Dvpq2sUu.js +10 -0
- package/dist/docs/assets/{reference_subagents.md.CAhvW4yS.lean.js → reference_subagents.md.Dvpq2sUu.lean.js} +1 -1
- package/dist/docs/assets/reference_tools.md.B2OaR1Nb.js +62 -0
- package/dist/docs/assets/{reference_tools.md.DQr7Fnhu.lean.js → reference_tools.md.B2OaR1Nb.lean.js} +1 -1
- package/dist/docs/assets/templates_agentic-owners.md.B6ZmKAzT.js +13 -0
- package/dist/docs/assets/{templates_agentic-owners.md.Dh2nDSqg.lean.js → templates_agentic-owners.md.B6ZmKAzT.lean.js} +1 -1
- package/dist/docs/assets/templates_reviews.md.dr9Xv_KM.js +10 -0
- package/dist/docs/assets/templates_reviews.md.dr9Xv_KM.lean.js +1 -0
- package/dist/docs/assets/troubleshooting.md.DfKKvW1R.js +3 -0
- package/dist/docs/assets/{troubleshooting.md.CGOdNVAL.lean.js → troubleshooting.md.DfKKvW1R.lean.js} +1 -1
- package/dist/docs/building-with-agents.html +41 -44
- package/dist/docs/building-with-agents.md +11 -104
- package/dist/docs/changelog.html +37 -37
- package/dist/docs/changelog.md +5 -155
- package/dist/docs/deployment.html +54 -54
- package/dist/docs/deployment.md +83 -262
- package/dist/docs/evals.html +52 -71
- package/dist/docs/evals.md +33 -129
- package/dist/docs/guides/agent-to-agent.html +43 -43
- package/dist/docs/guides/agent-to-agent.md +27 -64
- package/dist/docs/guides/bitbucket.html +62 -62
- package/dist/docs/guides/bitbucket.md +73 -134
- package/dist/docs/guides/cloud-agents.html +37 -37
- package/dist/docs/guides/cloud-agents.md +23 -53
- package/dist/docs/guides/convert-automation.html +39 -56
- package/dist/docs/guides/convert-automation.md +31 -115
- package/dist/docs/guides/github.html +46 -54
- package/dist/docs/guides/github.md +49 -131
- package/dist/docs/guides/gitlab.html +98 -108
- package/dist/docs/guides/gitlab.md +104 -184
- package/dist/docs/guides/improve.html +38 -43
- package/dist/docs/guides/improve.md +24 -90
- package/dist/docs/guides/jev.html +55 -95
- package/dist/docs/guides/jev.md +53 -160
- package/dist/docs/guides/mcp-oauth.html +38 -40
- package/dist/docs/guides/mcp-oauth.md +33 -67
- package/dist/docs/guides/migrate.html +38 -43
- package/dist/docs/guides/migrate.md +56 -130
- package/dist/docs/guides/opentelemetry.html +41 -43
- package/dist/docs/guides/opentelemetry.md +32 -27
- package/dist/docs/guides/pagerduty.html +50 -49
- package/dist/docs/guides/pagerduty.md +65 -199
- package/dist/docs/guides/slack.html +51 -57
- package/dist/docs/guides/slack.md +76 -204
- package/dist/docs/guides/webhooks.html +42 -42
- package/dist/docs/guides/webhooks.md +64 -146
- package/dist/docs/hashmap.json +1 -1
- package/dist/docs/hillclimbing.html +40 -42
- package/dist/docs/hillclimbing.md +40 -102
- package/dist/docs/index.html +55 -61
- package/dist/docs/index.md +49 -83
- package/dist/docs/llms-full.txt +2262 -5721
- package/dist/docs/llms.txt +9 -14
- package/dist/docs/quickstart.html +95 -152
- package/dist/docs/quickstart.md +91 -180
- package/dist/docs/reference/access.html +36 -36
- package/dist/docs/reference/access.md +7 -12
- package/dist/docs/reference/agent-config.html +43 -56
- package/dist/docs/reference/agent-config.md +91 -174
- package/dist/docs/reference/artifacts.html +38 -39
- package/dist/docs/reference/artifacts.md +38 -62
- package/dist/docs/reference/channels.html +38 -39
- package/dist/docs/reference/channels.md +102 -151
- package/dist/docs/reference/cli.html +68 -84
- package/dist/docs/reference/cli.md +232 -697
- package/dist/docs/reference/connections.html +45 -57
- package/dist/docs/reference/connections.md +83 -164
- package/dist/docs/reference/evals.html +47 -54
- package/dist/docs/reference/evals.md +75 -141
- package/dist/docs/reference/extensions.html +41 -43
- package/dist/docs/reference/extensions.md +37 -70
- package/dist/docs/reference/hooks.html +40 -52
- package/dist/docs/reference/hooks.md +62 -133
- package/dist/docs/reference/http-api.html +38 -42
- package/dist/docs/reference/http-api.md +66 -118
- package/dist/docs/reference/instructions.html +39 -39
- package/dist/docs/reference/instructions.md +19 -29
- package/dist/docs/reference/limits.html +38 -38
- package/dist/docs/reference/limits.md +9 -21
- package/dist/docs/reference/memory.html +37 -38
- package/dist/docs/reference/memory.md +38 -75
- package/dist/docs/reference/playground.html +36 -36
- package/dist/docs/reference/playground.md +14 -22
- package/dist/docs/reference/project-layout.html +49 -48
- package/dist/docs/reference/project-layout.md +38 -84
- package/dist/docs/reference/schedules.html +46 -48
- package/dist/docs/reference/schedules.md +59 -90
- package/dist/docs/reference/sessions.html +37 -37
- package/dist/docs/reference/sessions.md +66 -113
- package/dist/docs/reference/skills.html +39 -39
- package/dist/docs/reference/skills.md +11 -31
- package/dist/docs/reference/subagents.html +37 -37
- package/dist/docs/reference/subagents.md +17 -31
- package/dist/docs/reference/tools.html +47 -64
- package/dist/docs/reference/tools.md +48 -130
- package/dist/docs/templates/agentic-owners.html +38 -38
- package/dist/docs/templates/agentic-owners.md +50 -74
- package/dist/docs/templates/reviews.html +69 -0
- package/dist/docs/templates/reviews.md +71 -0
- package/dist/docs/troubleshooting.html +38 -36
- package/dist/docs/troubleshooting.md +48 -101
- package/dist/internal/cli-deploy.js +1 -1
- package/dist/internal/cli-docs.js +1 -1
- package/dist/internal/cli-images.js +1 -1
- package/dist/internal/cli-skills.js +1 -1
- package/dist/internal/cursor/credentials.js +1 -1
- package/dist/internal/deploy-manifest.d.ts +1 -0
- package/dist/internal/deploy-manifest.d.ts.map +1 -1
- package/dist/internal/deploy-manifest.js +0 -1
- package/dist/internal/discovery/agent-config.d.ts +2 -6
- package/dist/internal/discovery/agent-config.d.ts.map +1 -1
- package/dist/internal/discovery/agent-config.js +8 -25
- package/dist/internal/discovery/agent.d.ts.map +1 -1
- package/dist/internal/discovery/agent.js +2 -3
- package/dist/internal/discovery/project.d.ts.map +1 -1
- package/dist/internal/discovery/project.js +15 -0
- package/dist/internal/init-project.d.ts.map +1 -1
- package/dist/internal/init-project.js +7 -5
- package/dist/internal/reminder-runner.js +1 -1
- package/dist/internal/session-engine.js +1 -1
- package/dist/internal/shallow-clone.js +1 -1
- package/dist/playground/assets/index-D6mVn9og.js +62 -0
- package/dist/playground/assets/index-DlALM6gr.css +1 -0
- package/dist/playground/index.html +2 -2
- package/dist/types.d.ts +4 -17
- package/dist/types.d.ts.map +1 -1
- package/docs/README.md +49 -83
- package/docs/building-with-agents.md +11 -104
- package/docs/changelog.md +6 -156
- package/docs/deployment.md +83 -262
- package/docs/evals.md +34 -130
- package/docs/guides/agent-to-agent.md +27 -64
- package/docs/guides/bitbucket.md +73 -134
- package/docs/guides/cloud-agents.md +23 -53
- package/docs/guides/convert-automation.md +31 -115
- package/docs/guides/github.md +49 -131
- package/docs/guides/gitlab.md +104 -184
- package/docs/guides/improve.md +24 -90
- package/docs/guides/jev.md +53 -160
- package/docs/guides/mcp-oauth.md +33 -67
- package/docs/guides/migrate.md +56 -130
- package/docs/guides/opentelemetry.md +37 -27
- package/docs/guides/pagerduty.md +68 -200
- package/docs/guides/slack.md +76 -204
- package/docs/guides/webhooks.md +64 -146
- package/docs/hillclimbing.md +41 -103
- package/docs/quickstart.md +91 -180
- package/docs/reference/access.md +7 -12
- package/docs/reference/agent-config.md +91 -174
- package/docs/reference/artifacts.md +38 -62
- package/docs/reference/channels.md +102 -151
- package/docs/reference/cli.md +232 -697
- package/docs/reference/connections.md +83 -164
- package/docs/reference/evals.md +75 -141
- package/docs/reference/extensions.md +37 -70
- package/docs/reference/hooks.md +61 -132
- package/docs/reference/http-api.md +66 -118
- package/docs/reference/instructions.md +20 -30
- package/docs/reference/limits.md +10 -22
- package/docs/reference/memory.md +38 -75
- package/docs/reference/playground.md +14 -22
- package/docs/reference/project-layout.md +38 -84
- package/docs/reference/schedules.md +59 -90
- package/docs/reference/sessions.md +66 -113
- package/docs/reference/skills.md +12 -32
- package/docs/reference/subagents.md +17 -31
- package/docs/reference/tools.md +48 -130
- package/docs/templates/agentic-owners.md +50 -74
- package/docs/templates/reviews.md +76 -0
- package/docs/troubleshooting.md +48 -101
- package/package.json +2 -2
- package/skills/deploy/SKILL.md +1 -1
- package/skills/migrate-v2/SKILL.md +0 -1
- package/dist/docs/assets/building-with-agents.md.Bf1yH2tG.js +0 -9
- package/dist/docs/assets/changelog.md.DrLFQUlu.js +0 -1
- package/dist/docs/assets/changelog.md.DrLFQUlu.lean.js +0 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.Dpd_wxVU.js +0 -1
- package/dist/docs/assets/chunks/channel._fZX_hvU.js +0 -1
- package/dist/docs/assets/chunks/classDiagram-6PBFFD2Q.eClS1Q0E.js +0 -1
- package/dist/docs/assets/chunks/classDiagram-v2-HSJHXN6E.eClS1Q0E.js +0 -1
- package/dist/docs/assets/chunks/clone.W1Phdw9P.js +0 -1
- package/dist/docs/assets/chunks/stateDiagram-v2-QKLJ7IA2.77yf4kRr.js +0 -1
- package/dist/docs/assets/deployment.md.KW0Ni_rY.js +0 -31
- package/dist/docs/assets/evals.md.DiljU__Z.js +0 -68
- package/dist/docs/assets/evals.md.DiljU__Z.lean.js +0 -1
- package/dist/docs/assets/guides_agent-to-agent.md.D1D5f6Jc.lean.js +0 -1
- package/dist/docs/assets/guides_cloud-agents.md.Ceyjv0ul.js +0 -9
- package/dist/docs/assets/guides_convert-automation.md.BjraCvok.js +0 -43
- package/dist/docs/assets/guides_hooks.md.BjeEiM7s.js +0 -54
- package/dist/docs/assets/guides_hooks.md.BjeEiM7s.lean.js +0 -1
- package/dist/docs/assets/guides_improve.md.BFYtrtLE.js +0 -21
- package/dist/docs/assets/guides_migrate.md.RS9Eu-Xh.js +0 -8
- package/dist/docs/assets/guides_opentelemetry.md.CoZ1kNyd.js +0 -28
- package/dist/docs/assets/guides_opentelemetry.md.CoZ1kNyd.lean.js +0 -1
- package/dist/docs/assets/guides_pagerduty.md.mnKXe9U0.js +0 -60
- package/dist/docs/assets/guides_pagerduty.md.mnKXe9U0.lean.js +0 -1
- package/dist/docs/assets/guides_slack.md.BwXHURvY.js +0 -88
- package/dist/docs/assets/hillclimbing.md.By61f3uq.js +0 -7
- package/dist/docs/assets/hillclimbing.md.By61f3uq.lean.js +0 -1
- package/dist/docs/assets/index.md.CAruw_rp.js +0 -26
- package/dist/docs/assets/quickstart.md.CljDaJBi.js +0 -170
- package/dist/docs/assets/reference_access.md.CEtibb4i.js +0 -1
- package/dist/docs/assets/reference_access.md.CEtibb4i.lean.js +0 -1
- package/dist/docs/assets/reference_agent-config.md.Ck29qYNj.js +0 -40
- package/dist/docs/assets/reference_artifacts.md.C-hKR2hp.js +0 -18
- package/dist/docs/assets/reference_channels.md.DsJfGSBJ.js +0 -22
- package/dist/docs/assets/reference_cli.md.BkdLRbXg.js +0 -99
- package/dist/docs/assets/reference_cli.md.BkdLRbXg.lean.js +0 -1
- package/dist/docs/assets/reference_connections.md.BoAotr7_.js +0 -66
- package/dist/docs/assets/reference_evals.md.2BXeMiO2.js +0 -76
- package/dist/docs/assets/reference_hooks.md.B-bwJuiG.js +0 -51
- package/dist/docs/assets/reference_hooks.md.B-bwJuiG.lean.js +0 -1
- package/dist/docs/assets/reference_http-api.md._RWELr3d.js +0 -11
- package/dist/docs/assets/reference_instructions.md.yL2hXLnl.js +0 -14
- package/dist/docs/assets/reference_instructions.md.yL2hXLnl.lean.js +0 -1
- package/dist/docs/assets/reference_limits.md.BCCnvt3o.js +0 -8
- package/dist/docs/assets/reference_limits.md.BCCnvt3o.lean.js +0 -1
- package/dist/docs/assets/reference_memory.md.Dd7cyV9V.js +0 -4
- package/dist/docs/assets/reference_playground.md.CQiJk9CM.js +0 -1
- package/dist/docs/assets/reference_project-layout.md.BamcPDUn.js +0 -25
- package/dist/docs/assets/reference_project-layout.md.BamcPDUn.lean.js +0 -1
- package/dist/docs/assets/reference_prompt.md.BMjm1Jp3.js +0 -9
- package/dist/docs/assets/reference_prompt.md.BMjm1Jp3.lean.js +0 -1
- package/dist/docs/assets/reference_schedules.md.RC47aFm4.js +0 -47
- package/dist/docs/assets/reference_sessions.md.C7KFhWXV.js +0 -22
- package/dist/docs/assets/reference_subagents.md.CAhvW4yS.js +0 -10
- package/dist/docs/assets/reference_tools.md.DQr7Fnhu.js +0 -79
- package/dist/docs/assets/templates_agentic-owners.md.Dh2nDSqg.js +0 -13
- package/dist/docs/assets/templates_security-reviewer.md.DnwXdnB5.js +0 -3
- package/dist/docs/assets/templates_security-reviewer.md.DnwXdnB5.lean.js +0 -1
- package/dist/docs/assets/templates_thermo-quality-review.md.MJIdHkzE.js +0 -3
- package/dist/docs/assets/templates_thermo-quality-review.md.MJIdHkzE.lean.js +0 -1
- package/dist/docs/assets/templates_thermo-review.md.DfN1OFOl.js +0 -3
- package/dist/docs/assets/templates_thermo-review.md.DfN1OFOl.lean.js +0 -1
- package/dist/docs/assets/templates_triage.md.BYWMloV4.js +0 -9
- package/dist/docs/assets/templates_triage.md.BYWMloV4.lean.js +0 -1
- package/dist/docs/assets/troubleshooting.md.CGOdNVAL.js +0 -1
- package/dist/docs/guides/hooks.html +0 -113
- package/dist/docs/guides/hooks.md +0 -116
- package/dist/docs/package/skills/create-agent/SKILL.md +0 -178
- package/dist/docs/package/skills/evals/SKILL.md +0 -145
- package/dist/docs/package/skills/github/SKILL.md +0 -133
- package/dist/docs/package/skills/hillclimb/SKILL.md +0 -63
- package/dist/docs/package/skills/mcp-auth/SKILL.md +0 -60
- package/dist/docs/package/skills/migrate-v2/SKILL.md +0 -394
- package/dist/docs/package/skills/setup-slack/SKILL.md +0 -136
- package/dist/docs/reference/prompt.html +0 -68
- package/dist/docs/reference/prompt.md +0 -47
- package/dist/docs/templates/security-reviewer.html +0 -62
- package/dist/docs/templates/security-reviewer.md +0 -81
- package/dist/docs/templates/thermo-quality-review.html +0 -62
- package/dist/docs/templates/thermo-quality-review.md +0 -75
- package/dist/docs/templates/thermo-review.html +0 -62
- package/dist/docs/templates/thermo-review.md +0 -75
- package/dist/docs/templates/triage.html +0 -68
- package/dist/docs/templates/triage.md +0 -101
- package/dist/playground/assets/index-Cl9zvmmf.js +0 -62
- package/dist/playground/assets/index-F8b5WvWi.css +0 -1
- package/docs/guides/hooks.md +0 -121
- package/docs/reference/prompt.md +0 -52
- package/docs/templates/security-reviewer.md +0 -86
- package/docs/templates/thermo-quality-review.md +0 -80
- package/docs/templates/thermo-review.md +0 -80
- package/docs/templates/triage.md +0 -106
package/docs/reference/evals.md
CHANGED
|
@@ -6,23 +6,19 @@ description: "Define eval cases, drive turns, assert trajectories, and interpret
|
|
|
6
6
|
# Evals
|
|
7
7
|
|
|
8
8
|
Evals run fixed cases against an agent and record whether its turns,
|
|
9
|
-
tools, events, and output meet a contract.
|
|
10
|
-
|
|
11
|
-
discovers their IDs, executes them, and reports every assertion. See the
|
|
12
|
-
[Evals guide](../evals.md) for the regression workflow.
|
|
9
|
+
tools, events, and output meet a contract. The
|
|
10
|
+
[Evals guide](../evals.md) covers the regression workflow.
|
|
13
11
|
|
|
14
12
|
## Eval discovery and case IDs
|
|
15
13
|
|
|
16
14
|
Eval files live under the project-root `evals/` directory and end in
|
|
17
15
|
`.eval.ts` or `.eval.js`. The path under that directory becomes the
|
|
18
|
-
eval ID
|
|
16
|
+
eval ID, so `evals/readiness.eval.ts` becomes `readiness`,
|
|
17
|
+
`evals/builds/api.eval.ts` becomes `builds/api`, and
|
|
18
|
+
`evals/builds/index.eval.ts` becomes `builds`.
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
- `evals/builds/index.eval.ts` becomes `builds`.
|
|
23
|
-
|
|
24
|
-
Export one `defineEval` with either `test` or `cases`, not both. A case
|
|
25
|
-
ID appends one path segment to the file ID:
|
|
20
|
+
Export one `defineEval` that sets exactly one of `test` or `cases`. A
|
|
21
|
+
case ID adds one path segment to the file ID:
|
|
26
22
|
|
|
27
23
|
```ts
|
|
28
24
|
export default defineEval({
|
|
@@ -49,101 +45,87 @@ export default defineEval({
|
|
|
49
45
|
Those cases are `prs/checkout` and `prs/search` when the file is
|
|
50
46
|
`evals/prs.eval.ts`. Case IDs must be unique single path segments.
|
|
51
47
|
|
|
52
|
-
A file can also export an array of `defineEval` calls
|
|
53
|
-
|
|
54
|
-
handwritten scenarios stable IDs; arrays fit loaded datasets.
|
|
48
|
+
A file can also export an array of `defineEval` calls, named with
|
|
49
|
+
zero-padded indexes such as `sql/0000`.
|
|
55
50
|
|
|
56
|
-
`iterations` repeats one datapoint
|
|
57
|
-
`iterations: 3`, `weather/nyc`
|
|
58
|
-
`weather/nyc/
|
|
59
|
-
three.
|
|
51
|
+
`iterations` repeats one datapoint 1 to 100 times. With
|
|
52
|
+
`iterations: 3`, `weather/nyc` runs as `weather/nyc/1` through
|
|
53
|
+
`weather/nyc/3`, and selecting `weather/nyc` runs all three.
|
|
60
54
|
|
|
61
55
|
## Eval configuration
|
|
62
56
|
|
|
63
|
-
Every
|
|
64
|
-
`maxConcurrency`:
|
|
57
|
+
Every suite needs `evals/evals.config.ts` with `maxConcurrency`:
|
|
65
58
|
|
|
66
59
|
```ts
|
|
67
60
|
import { defineEvalConfig } from "@grok/agent-sdk/evals";
|
|
68
61
|
|
|
69
62
|
export default defineEvalConfig({
|
|
70
63
|
maxConcurrency: 20,
|
|
71
|
-
timeoutMs: 180_000,
|
|
72
64
|
judge: { model: "gpt-5.4-mini" },
|
|
73
65
|
});
|
|
74
66
|
```
|
|
75
67
|
|
|
76
68
|
| Option | Contract |
|
|
77
69
|
| --- | --- |
|
|
78
|
-
| `maxConcurrency` | Required
|
|
79
|
-
| `timeoutMs` | Per-case timeout
|
|
70
|
+
| `maxConcurrency` | Required. Concurrent authored datapoints, from 1 to 200 |
|
|
71
|
+
| `timeoutMs` | Per-case timeout. A case or file value wins, then the CLI flag, then this config |
|
|
80
72
|
| `judge` | Default model for `t.judge` |
|
|
81
73
|
| `reporters` | Objects notified as cases and runs complete |
|
|
82
|
-
| `maxPlaygroundRuns` |
|
|
74
|
+
| `maxPlaygroundRuns` | Server-side batches kept in playground history. Defaults to 20 |
|
|
83
75
|
|
|
84
76
|
A case can override `description`, `tags`, `timeoutMs`, `iterations`,
|
|
85
|
-
`judge`, `reporters`, and `metadata`. Case metadata merges over
|
|
86
|
-
|
|
77
|
+
`judge`, `reporters`, and `metadata`. Case metadata merges over file
|
|
78
|
+
metadata, and case reporters add to the file's reporters.
|
|
87
79
|
|
|
88
80
|
## Send turns in a case
|
|
89
81
|
|
|
90
82
|
`await t.send(message, options?)` runs one turn and waits until it
|
|
91
|
-
finishes or fails.
|
|
92
|
-
|
|
93
|
-
The returned turn exposes its assistant `message`, `sessionId`,
|
|
94
|
-
`events`, ordered `toolCalls`, `ok`, and one-based `index`. Assertions
|
|
95
|
-
on the turn inspect only that turn; assertions on `t` inspect the whole
|
|
96
|
-
case. Run values include `t.reply`, `t.events`, `t.turns`,
|
|
97
|
-
`t.sessionId`, `t.iteration`, `t.iterations`, and the timeout
|
|
98
|
-
`t.signal`.
|
|
83
|
+
finishes or fails. Sends in one test share a session.
|
|
99
84
|
|
|
100
|
-
|
|
85
|
+
The returned turn has the assistant `message`, `sessionId`, `events`,
|
|
86
|
+
ordered `toolCalls`, `ok`, and a one-based `index`. Assertions on a turn
|
|
87
|
+
inspect only that turn, and assertions on `t` inspect the whole case.
|
|
88
|
+
The case also exposes `t.reply`, `t.events`, `t.turns`, `t.sessionId`,
|
|
89
|
+
`t.iteration`, `t.iterations`, and the timeout `t.signal`.
|
|
101
90
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
| `workspaceDir` | Absolute working directory for a new local session |
|
|
91
|
+
On the first send only, `workspaceFiles` seeds the session workspace
|
|
92
|
+
from a map of relative paths to content, and `workspaceDir` sets an
|
|
93
|
+
absolute working directory for a new local session.
|
|
106
94
|
|
|
107
|
-
|
|
108
|
-
|
|
95
|
+
`turn.expectOk()` throws when the turn failed, so later steps that
|
|
96
|
+
depend on it don't run.
|
|
109
97
|
|
|
110
98
|
## Trajectory assertions
|
|
111
99
|
|
|
112
|
-
Assertions record failures without stopping the test
|
|
113
|
-
reports every violated contract.
|
|
100
|
+
Assertions record failures without stopping the test.
|
|
114
101
|
|
|
115
102
|
| Assertion | Checks |
|
|
116
103
|
| --- | --- |
|
|
117
104
|
| `t.succeeded()` | Run finished without failure |
|
|
118
|
-
| `t.calledTool(name, matcher?)` |
|
|
119
|
-
| `t.notCalledTool(name)` | No request for that tool occurred |
|
|
105
|
+
| `t.calledTool(name, matcher?)`, `t.notCalledTool(name)` | A matching tool request occurred, or none did |
|
|
120
106
|
| `t.loadedSkill(name)` | Agent opened a named skill |
|
|
121
|
-
| `t.toolOrder(names)` | Tool requests appeared in relative order |
|
|
122
|
-
| `t.usedNoTools()` | No tool was requested |
|
|
123
|
-
| `t.maxToolCalls(max)` | Tool-call count stayed under the bound |
|
|
107
|
+
| `t.toolOrder(names)`, `t.eventOrder(matchers)` | Tool requests or event groups appeared in relative order |
|
|
108
|
+
| `t.usedNoTools()`, `t.maxToolCalls(max)` | No tool was requested, or the count stayed under the bound |
|
|
124
109
|
| `t.noFailedActions()` | No tool result failed |
|
|
125
110
|
| `t.calledSubagent(name, matcher?)` | Matching subagent call occurred |
|
|
126
111
|
| `t.taggedArtifact(kind?, predicate?)` | Matching durable artifact was tagged |
|
|
127
|
-
| `t.event(type, matcher?)` |
|
|
128
|
-
| `t.notEvent(type, matcher?)` | Matching session event did not occur |
|
|
129
|
-
| `t.eventOrder(matchers)` | Event groups appeared in relative order |
|
|
112
|
+
| `t.event(type, matcher?)`, `t.notEvent(type, matcher?)` | A matching session event occurred, or none did |
|
|
130
113
|
| `t.eventsSatisfy(label, predicate)` | Predicate accepted the event stream |
|
|
131
114
|
| `t.check(value, expectation)` | Value met an expectation builder |
|
|
132
115
|
| `t.score(name, value)` | Recorded a numeric score from 0 to 1 |
|
|
133
116
|
|
|
134
117
|
`t.requireToolCall`, `t.requireInputRequest`, and `t.require` return the
|
|
135
|
-
matched value and stop the
|
|
118
|
+
matched value and stop the rest of the test when nothing matches.
|
|
136
119
|
|
|
137
120
|
## Tool and event matchers
|
|
138
121
|
|
|
139
122
|
Tool matchers can narrow by `input`, `output`, `status`, and `count`.
|
|
140
|
-
Values
|
|
141
|
-
literals
|
|
123
|
+
Values can be literals, regular expressions, or predicates. Object
|
|
124
|
+
literals match partially:
|
|
142
125
|
|
|
143
126
|
```ts
|
|
144
127
|
t.calledTool("inspect_pr", {
|
|
145
128
|
input: { verdict: "review" },
|
|
146
|
-
status: "completed",
|
|
147
129
|
count: 1,
|
|
148
130
|
});
|
|
149
131
|
```
|
|
@@ -154,43 +136,34 @@ Event matchers accept `data` and `count`.
|
|
|
154
136
|
|
|
155
137
|
## Expectation builders
|
|
156
138
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
| `includes(string | RegExp)` | Hard gate |
|
|
162
|
-
| `equals(value)` | Hard gate |
|
|
163
|
-
| `matches(schema)` | Hard gate |
|
|
164
|
-
| `satisfies(predicate, label)` | Hard gate |
|
|
165
|
-
| `similarity(expected)` | Soft score |
|
|
166
|
-
|
|
167
|
-
`matches` accepts Standard Schema implementations such as Zod.
|
|
139
|
+
`includes(string | RegExp)`, `equals(value)`, `matches(schema)`, and
|
|
140
|
+
`satisfies(predicate, label)` are hard gates by default, and
|
|
141
|
+
`similarity(expected)` is a soft score. `matches` accepts Standard
|
|
142
|
+
Schema implementations such as Zod.
|
|
168
143
|
`normalizedSimilarity(value, expected)` returns the same 0-to-1 value
|
|
169
|
-
|
|
144
|
+
that `similarity` uses.
|
|
170
145
|
|
|
171
146
|
## Severity and verdicts
|
|
172
147
|
|
|
173
148
|
Every assertion returns a severity handle:
|
|
174
149
|
|
|
175
150
|
```ts
|
|
176
|
-
t.succeeded();
|
|
177
151
|
t.calledTool("search").soft();
|
|
178
152
|
t.check(t.reply, similarity("Expected answer")).atLeast(0.8);
|
|
179
153
|
t.judge.closedQA("names the failing step").gate(1);
|
|
180
154
|
```
|
|
181
155
|
|
|
182
|
-
- `.gate(threshold?)` fails the case
|
|
183
|
-
- `.soft(threshold?)` records a value
|
|
184
|
-
- `.atLeast(threshold)`
|
|
185
|
-
verdict.
|
|
156
|
+
- `.gate(threshold?)` fails the case on a miss.
|
|
157
|
+
- `.soft(threshold?)` records a value and never fails the case.
|
|
158
|
+
- `.atLeast(threshold)` sets a soft bar, and a miss gives the case a
|
|
159
|
+
`scored` verdict.
|
|
186
160
|
|
|
187
|
-
Case verdicts are `passed`, `failed`, `scored`, or `skipped`.
|
|
188
|
-
|
|
189
|
-
set. Exit 2 means the selection matched no cases.
|
|
161
|
+
Case verdicts are `passed`, `failed`, `scored`, or `skipped`. A `scored`
|
|
162
|
+
case fails the run only under `--strict`.
|
|
190
163
|
|
|
191
164
|
## Judges
|
|
192
165
|
|
|
193
|
-
|
|
166
|
+
Judges grade what deterministic checks can't express:
|
|
194
167
|
|
|
195
168
|
```ts
|
|
196
169
|
t.judge.factuality(expected).atLeast(0.8);
|
|
@@ -199,26 +172,20 @@ t.judge.closedQA("names the root cause").gate(1);
|
|
|
199
172
|
t.judge.sql(expected).atLeast(0.9);
|
|
200
173
|
```
|
|
201
174
|
|
|
202
|
-
Each grades `t.reply`
|
|
203
|
-
|
|
204
|
-
call,
|
|
175
|
+
Each judge grades `t.reply` unless you pass `{ on }` with another value.
|
|
176
|
+
Set judge options in the project config, the eval, the case, or the
|
|
177
|
+
call, and the nearest value wins.
|
|
205
178
|
|
|
206
179
|
`t.judge.model(prompt)` returns a raw model reply for custom grading.
|
|
207
|
-
|
|
180
|
+
Agent output is untrusted input, so wrap it with `fenceUntrusted` and
|
|
208
181
|
include `EVAL_JUDGE_INJECTION_GUARD` in the judge prompt.
|
|
209
182
|
|
|
210
183
|
## Judge host
|
|
211
184
|
|
|
212
|
-
A judge
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
same case grades on macOS, a Linux cloud agent, and a Linux CI runner.
|
|
217
|
-
|
|
218
|
-
Without a credential, or when that credential belongs to a different
|
|
219
|
-
backend than the SDK will call, the case is skipped and the suite
|
|
220
|
-
continues. Without a judge model, the case fails and the assertion names
|
|
221
|
-
where to set one.
|
|
185
|
+
A judge needs a judge model and a Cursor credential from
|
|
186
|
+
`agent-sdk login`, `CURSOR_API_KEY`, or `CURSOR_SERVICE_ACCOUNT_KEY`.
|
|
187
|
+
Without a usable credential the runner skips the case, and without a
|
|
188
|
+
judge model it fails the case and names where to set one.
|
|
222
189
|
|
|
223
190
|
## Datasets and fixtures
|
|
224
191
|
|
|
@@ -228,16 +195,14 @@ Load committed JSON, JSONL, or YAML relative to the project root:
|
|
|
228
195
|
import { defineEval, includes } from "@grok/agent-sdk/evals";
|
|
229
196
|
import { loadYaml } from "@grok/agent-sdk/evals/loaders";
|
|
230
197
|
|
|
231
|
-
const rows = await loadYaml<
|
|
232
|
-
|
|
233
|
-
|
|
198
|
+
const rows = await loadYaml<Array<{ prompt: string; expected: string }>>(
|
|
199
|
+
"evals/data/cases.yaml",
|
|
200
|
+
);
|
|
234
201
|
|
|
235
202
|
export default rows.map(row =>
|
|
236
203
|
defineEval({
|
|
237
|
-
description: row.task,
|
|
238
204
|
async test(t) {
|
|
239
205
|
await t.send(row.prompt);
|
|
240
|
-
t.succeeded();
|
|
241
206
|
t.check(t.reply, includes(row.expected));
|
|
242
207
|
},
|
|
243
208
|
})
|
|
@@ -247,13 +212,18 @@ export default rows.map(row =>
|
|
|
247
212
|
## Reporters and results
|
|
248
213
|
|
|
249
214
|
Built-in reporters include `JUnit({ filePath, suiteName? })` and
|
|
250
|
-
`Artifacts({ dir })`.
|
|
215
|
+
`Artifacts({ dir })`. A custom reporter can implement `onRunStart`,
|
|
251
216
|
`onEvalComplete`, and `onRunComplete`.
|
|
252
217
|
|
|
253
|
-
JSON output
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
218
|
+
JSON output has run totals and one result per case. Each result has the
|
|
219
|
+
case ID, verdict, assertions, session ID, inputs, final text, tool
|
|
220
|
+
calls, metrics, logs, duration, metadata, tags, and any error or skip
|
|
221
|
+
reason.
|
|
222
|
+
|
|
223
|
+
Each local run writes `summary.json`, `results.jsonl`, and
|
|
224
|
+
`evals/<case-id>.json` under `evals/<stamp>/` in the project state
|
|
225
|
+
directory. `--state-root` doesn't move them, and `--artifacts <dir>` or
|
|
226
|
+
`--no-artifacts` does.
|
|
257
227
|
|
|
258
228
|
## Select cases
|
|
259
229
|
|
|
@@ -264,17 +234,14 @@ agent-sdk eval --dir . builds/checkout
|
|
|
264
234
|
agent-sdk eval --dir . --tag smoke
|
|
265
235
|
```
|
|
266
236
|
|
|
267
|
-
ID
|
|
268
|
-
|
|
269
|
-
|
|
237
|
+
An ID matches itself and its descendants. Repeated IDs or tags combine
|
|
238
|
+
with OR, and a case must match both groups when you pass both.
|
|
239
|
+
[Run evals](./cli.md#eval) covers runs on a server.
|
|
270
240
|
|
|
271
241
|
## Suite summary cases
|
|
272
242
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
aggregator by id, or with `--tag summary`, also runs every other case
|
|
276
|
-
from that file. A selection that includes no `summary` case stays as
|
|
277
|
-
you wrote it. `--list` shows the same set a run would execute.
|
|
243
|
+
A case tagged `EVAL_SUMMARY_TAG` aggregates its file, so selecting it by
|
|
244
|
+
ID or with `--tag summary` also runs every other case in that file.
|
|
278
245
|
|
|
279
246
|
```ts
|
|
280
247
|
import { defineEval, EVAL_SUMMARY_TAG } from "@grok/agent-sdk/evals";
|
|
@@ -299,36 +266,3 @@ export default defineEval({
|
|
|
299
266
|
],
|
|
300
267
|
});
|
|
301
268
|
```
|
|
302
|
-
|
|
303
|
-
When that file is `evals/prs.eval.ts`, `agent-sdk eval --dir . prs/overall`
|
|
304
|
-
runs `prs/checkout` and `prs/overall`.
|
|
305
|
-
|
|
306
|
-
## Inspect run artifacts
|
|
307
|
-
|
|
308
|
-
Default run artifacts go under `evals/<stamp>/` in the project state
|
|
309
|
-
directory and include
|
|
310
|
-
`summary.json`, `results.jsonl`, and `evals/<case-id>.json`.
|
|
311
|
-
`--artifacts <dir>` chooses another destination; `--no-artifacts`
|
|
312
|
-
disables them. This location is independent of `--state-root`.
|
|
313
|
-
|
|
314
|
-
## Run evals against a server
|
|
315
|
-
|
|
316
|
-
The playground **Evals** view starts batches on its running server, and
|
|
317
|
-
`--url` targets a named running server:
|
|
318
|
-
|
|
319
|
-
```text
|
|
320
|
-
agent-sdk eval --url http://127.0.0.1:3000/vulnerability-scanner --tag smoke
|
|
321
|
-
agent-sdk eval get <eval-id> --url http://127.0.0.1:3000/vulnerability-scanner
|
|
322
|
-
agent-sdk eval cancel <eval-id> --url http://127.0.0.1:3000/vulnerability-scanner
|
|
323
|
-
```
|
|
324
|
-
|
|
325
|
-
Server-side batches use the target's discovered cases and return an eval
|
|
326
|
-
ID for `status` or `cancel`. Local reporter and concurrency flags don't
|
|
327
|
-
apply to `--url` runs.
|
|
328
|
-
|
|
329
|
-
## Related
|
|
330
|
-
|
|
331
|
-
- [Evals guide](../evals.md)
|
|
332
|
-
- [CLI reference](./cli.md#eval)
|
|
333
|
-
- [Sessions](./sessions.md)
|
|
334
|
-
- [Artifacts](./artifacts.md)
|
|
@@ -6,20 +6,17 @@ description: "Mount a namespaced package of tools, skills, connections, and othe
|
|
|
6
6
|
# Extensions
|
|
7
7
|
|
|
8
8
|
An extension packages capabilities that would otherwise live under
|
|
9
|
-
`bot
|
|
10
|
-
|
|
11
|
-
adds them under a namespace without copying their source into your
|
|
12
|
-
project.
|
|
9
|
+
`bot/`, such as tools, skills, and connections. One mount file adds them
|
|
10
|
+
under a namespace without copying their source into your project.
|
|
13
11
|
|
|
14
|
-
Extensions and Cursor plugins
|
|
15
|
-
prevents name collisions; it does not sandbox the package or limit its
|
|
16
|
-
side effects. Review the package before mounting it.
|
|
12
|
+
Extensions and Cursor plugins run as trusted host code. A namespace prevents name collisions but doesn't sandbox the package, so review the package before mounting it.
|
|
17
13
|
|
|
18
14
|
## Mount an extension
|
|
19
15
|
|
|
20
16
|
Install the package and add one file under `bot/extensions/`. The
|
|
21
17
|
filename becomes the namespace, so this mount contributes names such as
|
|
22
|
-
`ci__failed_jobs` and `ci__triage`.
|
|
18
|
+
`ci__failed_jobs` and `ci__triage`. The `__` separator is reserved, so
|
|
19
|
+
root contributions can't impersonate a mounted one.
|
|
23
20
|
|
|
24
21
|
```ts
|
|
25
22
|
// bot/extensions/ci.ts
|
|
@@ -31,14 +28,13 @@ export default ci({
|
|
|
31
28
|
});
|
|
32
29
|
```
|
|
33
30
|
|
|
34
|
-
Pass `token` as a function so
|
|
35
|
-
|
|
31
|
+
Pass `token` as a function so it reads `BUILDKITE_TOKEN` when a tool
|
|
32
|
+
calls it.
|
|
36
33
|
|
|
37
|
-
The extension validates its
|
|
38
|
-
|
|
39
|
-
can call the contributed code.
|
|
34
|
+
The extension validates its settings at load, so a missing or mistyped
|
|
35
|
+
setting fails `agent-sdk validate`.
|
|
40
36
|
|
|
41
|
-
Extension instructions
|
|
37
|
+
Extension instructions follow the root agent's instructions.
|
|
42
38
|
The root agent still needs its own `bot/instructions.md`.
|
|
43
39
|
|
|
44
40
|
## Inspect a mount
|
|
@@ -50,24 +46,17 @@ agent-sdk validate --dir .
|
|
|
50
46
|
agent-sdk info --dir . --json
|
|
51
47
|
```
|
|
52
48
|
|
|
53
|
-
`info` lists each mount and
|
|
54
|
-
hooks, channels, schedules, subagents, and artifact kinds. The same
|
|
49
|
+
`info` lists each mount and every namespaced name it adds. The same
|
|
55
50
|
data appears under `extensions` in `GET /v1/info`.
|
|
56
51
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
Names inside the extension stay short. A tool authored as
|
|
62
|
-
`tools/failed_jobs.ts` appears to the mounting agent as
|
|
63
|
-
`<namespace>__failed_jobs`. The `__` separator is reserved, so root
|
|
64
|
-
contributions cannot impersonate a mounted one.
|
|
52
|
+
`info` also shows the egress hosts and secret names each mount adds to
|
|
53
|
+
a hosted deployment, and `agent-sdk deploy` names every extension that
|
|
54
|
+
adds egress.
|
|
65
55
|
|
|
66
56
|
## Adjust a mounted extension
|
|
67
57
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
extension's short name beneath it.
|
|
58
|
+
To replace or remove one contribution, mount as a directory. Put the
|
|
59
|
+
package mount in `extension.ts` and overlay the short name beneath it.
|
|
71
60
|
|
|
72
61
|
```text
|
|
73
62
|
bot/extensions/ci/
|
|
@@ -99,17 +88,16 @@ import { disableTool } from "@grok/agent-sdk/extensions";
|
|
|
99
88
|
export default disableTool();
|
|
100
89
|
```
|
|
101
90
|
|
|
102
|
-
The first file replaces `ci__job_log
|
|
91
|
+
The first file replaces `ci__job_log`, and the second removes
|
|
103
92
|
`ci__retry_job`. Use the matching `disable*` helper for another slot, or
|
|
104
93
|
generic `disable()` when the file location already identifies it.
|
|
105
|
-
Validation fails when an overlay names a contribution the package
|
|
106
|
-
|
|
94
|
+
Validation fails when an overlay names a contribution the package
|
|
95
|
+
doesn't provide.
|
|
107
96
|
|
|
108
97
|
## Mount a Cursor plugin
|
|
109
98
|
|
|
110
|
-
`cursorPlugin()`
|
|
111
|
-
|
|
112
|
-
skills, subagents, and connections.
|
|
99
|
+
`cursorPlugin()` mounts a vendored Cursor or Claude plugin the same way. Skills, commands, agents, and MCP servers become Agent
|
|
100
|
+
SDK skills, subagents, and connections.
|
|
113
101
|
|
|
114
102
|
```ts
|
|
115
103
|
// bot/extensions/notion.ts
|
|
@@ -123,16 +111,15 @@ export default cursorPlugin({
|
|
|
123
111
|
});
|
|
124
112
|
```
|
|
125
113
|
|
|
126
|
-
`path` is relative to the project root. `variables` fills
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
114
|
+
`path` is relative to the project root. `variables` fills the plugin's
|
|
115
|
+
MCP placeholders with `${NAME}` templates read when the connection
|
|
116
|
+
opens. An unresolved placeholder fails
|
|
117
|
+
validation instead of mounting a partial server.
|
|
130
118
|
|
|
131
|
-
Plugin rules and hooks
|
|
119
|
+
Plugin rules and hooks aren't mounted. Overlay a generated
|
|
132
120
|
`mcp-connections/<server>.ts` file to replace or disable one plugin
|
|
133
121
|
server. OAuth-backed plugin servers use
|
|
134
|
-
[Host MCP OAuth](../guides/mcp-oauth.md)
|
|
135
|
-
details live in [MCP connections](./connections.md).
|
|
122
|
+
[Host MCP OAuth](../guides/mcp-oauth.md).
|
|
136
123
|
|
|
137
124
|
## Extension contents
|
|
138
125
|
|
|
@@ -147,8 +134,8 @@ details live in [MCP connections](./connections.md).
|
|
|
147
134
|
## Build an extension
|
|
148
135
|
|
|
149
136
|
Lay out the package like the shareable part of `bot/`. The root
|
|
150
|
-
`extension.ts` supplies a description and validates
|
|
151
|
-
|
|
137
|
+
`extension.ts` supplies a description and validates the consumer's
|
|
138
|
+
settings.
|
|
152
139
|
|
|
153
140
|
```text
|
|
154
141
|
@acme/ci-triage/
|
|
@@ -160,8 +147,6 @@ consumer must provide.
|
|
|
160
147
|
job_log.ts
|
|
161
148
|
skills/
|
|
162
149
|
triage.md
|
|
163
|
-
hooks/
|
|
164
|
-
audit.ts
|
|
165
150
|
lib/
|
|
166
151
|
buildkite.ts
|
|
167
152
|
```
|
|
@@ -186,29 +171,11 @@ export default defineExtension({
|
|
|
186
171
|
});
|
|
187
172
|
```
|
|
188
173
|
|
|
189
|
-
|
|
190
|
-
`extension.config`,
|
|
191
|
-
|
|
192
|
-
prose assets beside it.
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
whatever it returns, so a hosted deploy reaches Buildkite without the
|
|
198
|
-
consumer listing `api.buildkite.com`. List `BUILDKITE_TOKEN` in
|
|
199
|
-
`secretNames`. Call `extension.config.token()` inside a tool.
|
|
200
|
-
|
|
201
|
-
The [Project layout](./project-layout.md) owns each contribution's
|
|
202
|
-
authoring contract. Keep extension instructions and tool descriptions
|
|
203
|
-
namespace-neutral because the consumer chooses the final prefix.
|
|
204
|
-
|
|
205
|
-
## Related
|
|
206
|
-
|
|
207
|
-
- [Cloud agents](../guides/cloud-agents.md): delegate repository work
|
|
208
|
-
through an extension
|
|
209
|
-
- [Jev](../guides/jev.md) (experimental): typed answers, then gated
|
|
210
|
-
writes
|
|
211
|
-
- [Self-improvement](../guides/improve.md): propose source changes
|
|
212
|
-
through pull requests
|
|
213
|
-
- [Project layout](./project-layout.md): contribution slots
|
|
214
|
-
- [Tools](./tools.md) and [Skills](./skills.md): capability contracts
|
|
174
|
+
Package modules import this definition and read the validated
|
|
175
|
+
`extension.config`, with defaults applied. `hosting` adds its hosts and secret names to the mounting
|
|
176
|
+
agent's deployment. Publish the built entry point with its slot
|
|
177
|
+
directories and prose assets beside it.
|
|
178
|
+
|
|
179
|
+
Keep extension instructions and tool descriptions namespace-neutral
|
|
180
|
+
because the consumer chooses the prefix. [Project layout](./project-layout.md)
|
|
181
|
+
covers each slot's contract.
|