@cursor/july 0.1.16 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -4
- package/dist/bin/agent-serve.js +60 -2
- package/dist/channels/slack/api.d.ts.map +1 -1
- package/dist/channels/slack/api.js +13 -3
- package/dist/channels/slack/auth.d.ts +7 -1
- package/dist/channels/slack/auth.d.ts.map +1 -1
- package/dist/channels/slack/auth.js +14 -0
- package/dist/channels/slack/blocks.d.ts +61 -0
- package/dist/channels/slack/blocks.d.ts.map +1 -0
- package/dist/channels/slack/blocks.js +90 -0
- package/dist/channels/slack/defaults.d.ts +2 -0
- package/dist/channels/slack/defaults.d.ts.map +1 -1
- package/dist/channels/slack/defaults.js +29 -3
- package/dist/channels/slack/dispatch.d.ts +3 -17
- package/dist/channels/slack/dispatch.d.ts.map +1 -1
- package/dist/channels/slack/index.d.ts +1 -0
- package/dist/channels/slack/index.d.ts.map +1 -1
- package/dist/channels/slack/index.js +1 -0
- package/dist/channels/slack/interactive.d.ts +39 -3
- package/dist/channels/slack/interactive.d.ts.map +1 -1
- package/dist/channels/slack/interactive.js +152 -7
- package/dist/channels/slack/manifest.d.ts +3 -0
- package/dist/channels/slack/manifest.d.ts.map +1 -1
- package/dist/channels/slack/manifest.js +3 -0
- package/dist/channels/slack/setup.d.ts +4 -1
- package/dist/channels/slack/setup.d.ts.map +1 -1
- package/dist/channels/slack/setup.js +16 -2
- package/dist/channels/slack/slack-channel.d.ts.map +1 -1
- package/dist/channels/slack/slack-channel.js +24 -3
- package/dist/channels/slack/types.d.ts +77 -0
- package/dist/channels/slack/types.d.ts.map +1 -1
- package/dist/connections.d.ts +4 -1
- package/dist/connections.d.ts.map +1 -1
- package/dist/connections.js +4 -1
- package/dist/docs/404.html +2 -2
- package/dist/docs/ab.html +3 -3
- package/dist/docs/assets/{app.DjHC-Ggf.js → app.BbEXr2KO.js} +1 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.DCBsrfwP.js +1 -0
- package/dist/docs/assets/chunks/{VPLocalSearchBox.DwviwmIq.js → VPLocalSearchBox.DBEi4HDp.js} +1 -1
- package/dist/docs/assets/chunks/{theme.BIpeYSon.js → theme.DGfptcHw.js} +2 -2
- package/dist/docs/assets/{guides_mcp-oauth.md.DqmEOQY1.js → guides_mcp-oauth.md.Dd8EgSem.js} +3 -2
- package/dist/docs/assets/{guides_mcp-oauth.md.DqmEOQY1.lean.js → guides_mcp-oauth.md.Dd8EgSem.lean.js} +1 -1
- package/dist/docs/assets/{guides_slack.md.FWale6Ip.js → guides_slack.md.D_xFKlnF.js} +1 -1
- package/dist/docs/assets/{reference_connections.md.CsSr3rDz.js → reference_connections.md.B9Q3TOve.js} +12 -5
- package/dist/docs/assets/{reference_connections.md.CsSr3rDz.lean.js → reference_connections.md.B9Q3TOve.lean.js} +1 -1
- package/dist/docs/building-with-agents.html +3 -3
- package/dist/docs/concepts.html +3 -3
- package/dist/docs/deployment.html +3 -3
- package/dist/docs/evals.html +3 -3
- package/dist/docs/example-agents/approval-buddy.html +3 -3
- package/dist/docs/example-agents/benny.html +3 -3
- package/dist/docs/example-agents/bugbot.html +3 -3
- package/dist/docs/example-agents/codebase-wiki.html +3 -3
- package/dist/docs/example-agents/codeowners-review.html +3 -3
- package/dist/docs/example-agents/concierge.html +3 -3
- package/dist/docs/example-agents/fsd.html +3 -3
- package/dist/docs/example-agents/index.html +3 -3
- package/dist/docs/example-agents/knowledge-base.html +3 -3
- package/dist/docs/example-agents/oncall.html +3 -3
- package/dist/docs/example-agents/security-reviewer.html +3 -3
- package/dist/docs/example-agents/slack-agent.html +3 -3
- package/dist/docs/example-agents/weather-agent.html +3 -3
- package/dist/docs/guides/agent-to-agent.html +3 -3
- package/dist/docs/guides/cloud-runtime.html +3 -3
- package/dist/docs/guides/github.html +3 -3
- package/dist/docs/guides/human-in-the-loop.html +3 -3
- package/dist/docs/guides/mcp-oauth.html +6 -5
- package/dist/docs/guides/slack.html +5 -5
- package/dist/docs/guides/webhooks.html +3 -3
- package/dist/docs/hashmap.json +1 -1
- package/dist/docs/hillclimbing.html +3 -3
- package/dist/docs/index.html +3 -3
- package/dist/docs/quickstart.html +3 -3
- package/dist/docs/reference/agent-config.html +3 -3
- package/dist/docs/reference/channels.html +3 -3
- package/dist/docs/reference/cli.html +3 -3
- package/dist/docs/reference/connections.html +14 -7
- package/dist/docs/reference/hooks.html +3 -3
- package/dist/docs/reference/http-api.html +3 -3
- package/dist/docs/reference/instructions.html +3 -3
- package/dist/docs/reference/playground.html +3 -3
- package/dist/docs/reference/project-layout.html +3 -3
- package/dist/docs/reference/prompt.html +3 -3
- package/dist/docs/reference/schedules.html +3 -3
- package/dist/docs/reference/sessions.html +3 -3
- package/dist/docs/reference/skills.html +3 -3
- package/dist/docs/reference/subagents.html +3 -3
- package/dist/docs/reference/tools.html +3 -3
- package/dist/docs/scaffolding-agents.html +3 -3
- package/dist/docs/storage.html +3 -3
- package/dist/docs/troubleshooting.html +3 -3
- package/dist/evals/assertions.d.ts +190 -0
- package/dist/evals/assertions.d.ts.map +1 -0
- package/dist/evals/assertions.js +461 -0
- package/dist/evals/expect.d.ts +95 -0
- package/dist/evals/expect.d.ts.map +1 -0
- package/dist/evals/expect.js +194 -0
- package/dist/evals/judge.d.ts +133 -0
- package/dist/evals/judge.d.ts.map +1 -0
- package/dist/evals/judge.js +228 -0
- package/dist/evals/loaders.d.ts +51 -0
- package/dist/evals/loaders.d.ts.map +1 -0
- package/dist/evals/loaders.js +124 -0
- package/dist/evals/matchers.d.ts +95 -0
- package/dist/evals/matchers.d.ts.map +1 -0
- package/dist/evals/matchers.js +145 -0
- package/dist/evals/reporters.d.ts +70 -0
- package/dist/evals/reporters.d.ts.map +1 -0
- package/dist/evals/reporters.js +221 -0
- package/dist/evals/results.d.ts +103 -0
- package/dist/evals/results.d.ts.map +1 -0
- package/dist/evals/results.js +28 -0
- package/dist/evals/run-facts.d.ts +77 -0
- package/dist/evals/run-facts.d.ts.map +1 -0
- package/dist/evals/run-facts.js +187 -0
- package/dist/evals.d.ts +118 -31
- package/dist/evals.d.ts.map +1 -1
- package/dist/evals.js +46 -12
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/internal/cli-ax.d.ts +27 -0
- package/dist/internal/cli-ax.d.ts.map +1 -1
- package/dist/internal/cli-ax.js +90 -12
- package/dist/internal/cli-mcp.d.ts +8 -0
- package/dist/internal/cli-mcp.d.ts.map +1 -1
- package/dist/internal/cli-mcp.js +98 -0
- package/dist/internal/cli-slack.d.ts +29 -1
- package/dist/internal/cli-slack.d.ts.map +1 -1
- package/dist/internal/cli-slack.js +777 -12
- package/dist/internal/deploy-client.d.ts +7 -0
- package/dist/internal/deploy-client.d.ts.map +1 -1
- package/dist/internal/deploy-client.js +9 -0
- package/dist/internal/discovery.d.ts.map +1 -1
- package/dist/internal/discovery.js +29 -9
- package/dist/internal/distribution.d.ts.map +1 -1
- package/dist/internal/distribution.js +2 -0
- package/dist/internal/eval-judge-model.d.ts +59 -0
- package/dist/internal/eval-judge-model.d.ts.map +1 -0
- package/dist/internal/eval-judge-model.js +131 -0
- package/dist/internal/eval-run-store.d.ts.map +1 -1
- package/dist/internal/eval-run-store.js +12 -20
- package/dist/internal/eval-runner.d.ts +31 -25
- package/dist/internal/eval-runner.d.ts.map +1 -1
- package/dist/internal/eval-runner.js +329 -126
- package/dist/internal/evals-client.d.ts.map +1 -1
- package/dist/internal/evals-client.js +4 -1
- package/dist/internal/local-env.d.ts +8 -0
- package/dist/internal/local-env.d.ts.map +1 -0
- package/dist/internal/local-env.js +47 -0
- package/dist/internal/playground/static.d.ts +3 -0
- package/dist/internal/playground/static.d.ts.map +1 -1
- package/dist/internal/playground/static.js +3 -0
- package/dist/internal/run-client.d.ts +15 -0
- package/dist/internal/run-client.d.ts.map +1 -1
- package/dist/internal/run-client.js +3 -2
- package/dist/internal/sdk-runner.d.ts +36 -1
- package/dist/internal/sdk-runner.d.ts.map +1 -1
- package/dist/internal/sdk-runner.js +65 -25
- package/dist/internal/server.d.ts.map +1 -1
- package/dist/internal/server.js +10 -1
- package/dist/internal/session-engine.d.ts +18 -5
- package/dist/internal/session-engine.d.ts.map +1 -1
- package/dist/internal/session-engine.js +72 -19
- package/dist/internal/slack-provision-client.d.ts +130 -0
- package/dist/internal/slack-provision-client.d.ts.map +1 -0
- package/dist/internal/slack-provision-client.js +284 -0
- package/dist/playground/assets/cursor-icons-16-f_W_ogc-.woff2 +0 -0
- package/dist/playground/assets/index-CidizGZv.css +1 -0
- package/dist/playground/assets/index-DNqirsrK.js +85 -0
- package/dist/playground/index.html +2 -2
- package/dist/types.d.ts +70 -4
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +26 -0
- package/docs/guides/mcp-oauth.md +10 -5
- package/docs/guides/slack.md +3 -2
- package/docs/reference/connections.md +28 -18
- package/package.json +20 -2
- package/skills/create-agent/SKILL.md +15 -7
- package/skills/debug/SKILL.md +3 -1
- package/skills/mcp-auth/SKILL.md +1 -1
- package/skills/setup-slack/SKILL.md +173 -11
- package/src/bin/agent-serve.ts +63 -2
- package/src/channels/slack/api.ts +16 -3
- package/src/channels/slack/auth.ts +16 -1
- package/src/channels/slack/blocks.ts +139 -0
- package/src/channels/slack/defaults.ts +32 -4
- package/src/channels/slack/dispatch.ts +3 -18
- package/src/channels/slack/index.ts +1 -0
- package/src/channels/slack/interactive.ts +183 -19
- package/src/channels/slack/manifest.ts +3 -0
- package/src/channels/slack/setup.ts +16 -2
- package/src/channels/slack/slack-channel.ts +37 -6
- package/src/channels/slack/types.ts +89 -0
- package/src/connections.ts +4 -1
- package/src/evals/assertions.ts +704 -0
- package/src/evals/expect.ts +272 -0
- package/src/evals/judge.ts +346 -0
- package/src/evals/loaders.ts +113 -0
- package/src/evals/matchers.ts +246 -0
- package/src/evals/reporters.ts +274 -0
- package/src/evals/results.ts +132 -0
- package/src/evals/run-facts.ts +274 -0
- package/src/evals.ts +205 -42
- package/src/index.ts +2 -0
- package/src/internal/cli-ax.ts +134 -15
- package/src/internal/cli-mcp.ts +118 -5
- package/src/internal/cli-slack.ts +993 -15
- package/src/internal/deploy-client.ts +10 -0
- package/src/internal/discovery.ts +31 -5
- package/src/internal/distribution.ts +2 -0
- package/src/internal/eval-judge-model.ts +148 -0
- package/src/internal/eval-run-store.ts +10 -14
- package/src/internal/eval-runner.ts +460 -214
- package/src/internal/evals-client.ts +5 -0
- package/src/internal/local-env.ts +53 -0
- package/src/internal/playground/static.ts +3 -0
- package/src/internal/run-client.ts +17 -1
- package/src/internal/sdk-runner.ts +103 -22
- package/src/internal/server.ts +10 -0
- package/src/internal/session-engine.ts +89 -21
- package/src/internal/slack-provision-client.ts +438 -0
- package/src/types.ts +92 -4
- package/dist/channels/slack/post-update-delivery.d.ts +0 -85
- package/dist/channels/slack/post-update-delivery.d.ts.map +0 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.DZtu0ZIc.js +0 -1
- package/dist/internal/json-dir-store.d.ts +0 -32
- package/dist/internal/json-dir-store.d.ts.map +0 -1
- package/dist/internal/persistence-coordinator.d.ts +0 -127
- package/dist/internal/persistence-coordinator.d.ts.map +0 -1
- package/dist/persistence.d.ts +0 -184
- package/dist/persistence.d.ts.map +0 -1
- package/dist/playground/assets/cursor-icons-16-CQ50JpfO.woff2 +0 -0
- package/dist/playground/assets/index-Bwl4Onx4.js +0 -85
- package/dist/playground/assets/index-ByV4nfch.css +0 -1
- /package/dist/docs/assets/{guides_slack.md.FWale6Ip.lean.js → guides_slack.md.D_xFKlnF.lean.js} +0 -0
|
@@ -191,6 +191,13 @@ export declare class AgentServeDeployClient {
|
|
|
191
191
|
}): Promise<string>;
|
|
192
192
|
/** Full URL of the stable engine alias for a deployment. */
|
|
193
193
|
aliasUrl(engineAlias: string): string;
|
|
194
|
+
/**
|
|
195
|
+
* Stable remote MCP gateway URL for a deployment, on the user-API-key
|
|
196
|
+
* surface (headless / CLI, `agentkit login` credential). The gateway
|
|
197
|
+
* mints/refreshes engineAccess server-side and proxies `/v1/mcp` to the
|
|
198
|
+
* pod, so a signed-in client never sees a pod hostname or port token.
|
|
199
|
+
*/
|
|
200
|
+
mcpGatewayUrl(teamId: number, slug: string): string;
|
|
194
201
|
private request;
|
|
195
202
|
}
|
|
196
203
|
//# sourceMappingURL=deploy-client.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"deploy-client.d.ts","sourceRoot":"","sources":["../../src/internal/deploy-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAUH,MAAM,MAAM,gBAAgB,GACxB,SAAS,GACT,WAAW,GACX,SAAS,GACT,UAAU,GACV,QAAQ,GACR,UAAU,GACV,SAAS,CAAC;AAEd,sEAAsE;AACtE,MAAM,MAAM,kBAAkB,GAC1B,QAAQ,GACR,WAAW,GACX,UAAU,GACV,UAAU,GACV,UAAU,GACV,YAAY,GACZ,SAAS,GACT,QAAQ,GACR,UAAU,GACV,SAAS,GACT,UAAU,CAAC;AAEf,eAAO,MAAM,wBAAwB,EAAE,WAAW,CAAC,gBAAgB,CAKjE,CAAC;AAEH;;;;;;;GAOG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,SAAS,CAAC;IAChB,oEAAoE;IACpE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uEAAuE;IACvE,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;CACjC,CAAC;AAIF,0EAA0E;AAC1E,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAKvE;AAED,6EAA6E;AAC7E,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOtE;AAED,qBAAa,cAAe,SAAQ,KAAK;IAGrC,QAAQ,CAAC,MAAM,EAAE,MAAM;IACvB,oEAAoE;IACpE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM;IAJrC,YACE,OAAO,EAAE,MAAM,EACN,MAAM,EAAE,MAAM;IACvB,oEAAoE;IAC3D,iBAAiB,CAAC,EAAE,MAAM,YAAA,EAIpC;CACF;AASD,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,gBAAgB,CAAC;IACzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,mEAAmE;IACnE,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,sDAAsD;IACtD,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC,wDAAwD;IACxD,gBAAgB,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAC1C,4DAA4D;IAC5D,sBAAsB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,0EAA0E;IAC1E,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAiB,SAAQ,UAAU;IAClD,YAAY,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;CACpC;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CACxC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,kBAAkB,CAAC,GAC/C,MAAM,GAAG,SAAS,CAOpB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,+EAA+E;AAC/E,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAgID,MAAM,WAAW,6BAA6B;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,2CAA2C;IAC3C,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B;AAED,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IAEzC,YAAY,OAAO,EAAE,6BAA6B,EAMjD;IAED,yEAAyE;IACnE,MAAM,CAAC,IAAI,EAAE;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,YAAY,CAAC;KACtB,GAAG,OAAO,CAAC,cAAc,CAAC,CAuC1B;IAEK,eAAe,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAOrE;IAEK,aAAa,CAAC,IAAI,EAAE;QACxB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;KACd,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAO5B;IAEK,cAAc,CAAC,IAAI,EAAE;QACzB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;KACd,GAAG,OAAO,CAAC,YAAY,CAAC,CAOxB;IAED,sEAAsE;IAChE,gBAAgB,CAAC,IAAI,EAAE;QAC3B,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;KACd,GAAG,OAAO,CAAC,MAAM,CAAC,CAOlB;IAED;;;;OAIG;IACG,UAAU,CAAC,IAAI,EAAE;QACrB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACjD,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CA0BpB;IAED,gEAAgE;IAC1D,WAAW,CAAC,IAAI,EAAE;QACtB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;KACd,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAOxB;IAED,yDAAyD;IACnD,YAAY,CAAC,IAAI,EAAE;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;KACd,GAAG,OAAO,CAAC,IAAI,CAAC,CAUhB;IAED;;;;OAIG;IACG,mBAAmB,CAAC,IAAI,EAAE;QAC9B,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;KACd,GAAG,OAAO,CAAC,MAAM,CAAC,CAOlB;IAED,4DAA4D;IAC5D,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAEpC;YAEa,OAAO;CAyCtB"}
|
|
1
|
+
{"version":3,"file":"deploy-client.d.ts","sourceRoot":"","sources":["../../src/internal/deploy-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAUH,MAAM,MAAM,gBAAgB,GACxB,SAAS,GACT,WAAW,GACX,SAAS,GACT,UAAU,GACV,QAAQ,GACR,UAAU,GACV,SAAS,CAAC;AAEd,sEAAsE;AACtE,MAAM,MAAM,kBAAkB,GAC1B,QAAQ,GACR,WAAW,GACX,UAAU,GACV,UAAU,GACV,UAAU,GACV,YAAY,GACZ,SAAS,GACT,QAAQ,GACR,UAAU,GACV,SAAS,GACT,UAAU,CAAC;AAEf,eAAO,MAAM,wBAAwB,EAAE,WAAW,CAAC,gBAAgB,CAKjE,CAAC;AAEH;;;;;;;GAOG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,SAAS,CAAC;IAChB,oEAAoE;IACpE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uEAAuE;IACvE,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;CACjC,CAAC;AAIF,0EAA0E;AAC1E,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAKvE;AAED,6EAA6E;AAC7E,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOtE;AAED,qBAAa,cAAe,SAAQ,KAAK;IAGrC,QAAQ,CAAC,MAAM,EAAE,MAAM;IACvB,oEAAoE;IACpE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM;IAJrC,YACE,OAAO,EAAE,MAAM,EACN,MAAM,EAAE,MAAM;IACvB,oEAAoE;IAC3D,iBAAiB,CAAC,EAAE,MAAM,YAAA,EAIpC;CACF;AASD,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,gBAAgB,CAAC;IACzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,mEAAmE;IACnE,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,sDAAsD;IACtD,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC,wDAAwD;IACxD,gBAAgB,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAC1C,4DAA4D;IAC5D,sBAAsB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,0EAA0E;IAC1E,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,gEAAgE;AAChE,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAiB,SAAQ,UAAU;IAClD,YAAY,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;CACpC;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CACxC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,kBAAkB,CAAC,GAC/C,MAAM,GAAG,SAAS,CAOpB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,+EAA+E;AAC/E,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAgID,MAAM,WAAW,6BAA6B;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,2CAA2C;IAC3C,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B;AAED,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IAEzC,YAAY,OAAO,EAAE,6BAA6B,EAMjD;IAED,yEAAyE;IACnE,MAAM,CAAC,IAAI,EAAE;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,YAAY,CAAC;KACtB,GAAG,OAAO,CAAC,cAAc,CAAC,CAuC1B;IAEK,eAAe,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAOrE;IAEK,aAAa,CAAC,IAAI,EAAE;QACxB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;KACd,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAO5B;IAEK,cAAc,CAAC,IAAI,EAAE;QACzB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;KACd,GAAG,OAAO,CAAC,YAAY,CAAC,CAOxB;IAED,sEAAsE;IAChE,gBAAgB,CAAC,IAAI,EAAE;QAC3B,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;KACd,GAAG,OAAO,CAAC,MAAM,CAAC,CAOlB;IAED;;;;OAIG;IACG,UAAU,CAAC,IAAI,EAAE;QACrB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACjD,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CA0BpB;IAED,gEAAgE;IAC1D,WAAW,CAAC,IAAI,EAAE;QACtB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;KACd,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAOxB;IAED,yDAAyD;IACnD,YAAY,CAAC,IAAI,EAAE;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;KACd,GAAG,OAAO,CAAC,IAAI,CAAC,CAUhB;IAED;;;;OAIG;IACG,mBAAmB,CAAC,IAAI,EAAE;QAC9B,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;KACd,GAAG,OAAO,CAAC,MAAM,CAAC,CAOlB;IAED,4DAA4D;IAC5D,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAEpC;IAED;;;;;OAKG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAElD;YAEa,OAAO;CAyCtB"}
|
|
@@ -334,6 +334,15 @@ export class AgentServeDeployClient {
|
|
|
334
334
|
aliasUrl(engineAlias) {
|
|
335
335
|
return `${this.backendUrl}${engineAlias}`;
|
|
336
336
|
}
|
|
337
|
+
/**
|
|
338
|
+
* Stable remote MCP gateway URL for a deployment, on the user-API-key
|
|
339
|
+
* surface (headless / CLI, `agentkit login` credential). The gateway
|
|
340
|
+
* mints/refreshes engineAccess server-side and proxies `/v1/mcp` to the
|
|
341
|
+
* pod, so a signed-in client never sees a pod hostname or port token.
|
|
342
|
+
*/
|
|
343
|
+
mcpGatewayUrl(teamId, slug) {
|
|
344
|
+
return `${this.backendUrl}/internal/agent-serve/mcp/${teamId}/${encodeURIComponent(slug)}`;
|
|
345
|
+
}
|
|
337
346
|
request(args) {
|
|
338
347
|
return __awaiter(this, void 0, void 0, function* () {
|
|
339
348
|
let response;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"discovery.d.ts","sourceRoot":"","sources":["../../src/internal/discovery.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAgBH,OAAO,EAIL,KAAK,YAAY,EACjB,KAAK,gBAAgB,
|
|
1
|
+
{"version":3,"file":"discovery.d.ts","sourceRoot":"","sources":["../../src/internal/discovery.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAgBH,OAAO,EAIL,KAAK,YAAY,EACjB,KAAK,gBAAgB,EAMrB,KAAK,UAAU,EAqBhB,MAAM,aAAa,CAAC;AAqBrB,MAAM,WAAW,uBAAuB;IACtC,sDAAsD;IACtD,YAAY,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CAC3E;AAmBD;;;;GAIG;AACH,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CASrE;AAED,sEAAsE;AACtE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;GAIG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,aAAa,EAAE,CAAC,CA2B1B;AAED;;;;;;GAMG;AACH,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,uBAA4B,GACpC,OAAO,CAAC,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,YAAY,CAAA;CAAE,CAAC,CAAC,CAoBzD;AAED;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,uBAAuB,GAAG;IACjC,IAAI,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC;CACtB,GACL,OAAO,CAAC,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,YAAY,CAAA;CAAE,CAAC,CAAC,CAgBzD;AAWD,iFAAiF;AACjF,eAAO,MAAM,cAAc,EAAE,GAAG,CAAC,MAAM,CAIrC,CAAC;AAEH,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,uBAA4B,GACpC,OAAO,CAAC,YAAY,CAAC,CA8DvB;AAED,qEAAqE;AACrE,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAElE;AAED,wBAAgB,iBAAiB,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,MAAM,CAOnE;AA8nBD;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAiB3E;AAgmBD,wBAAgB,WAAW,CAAC,OAAO,EAAE,YAAY,GAAG,gBAAgB,CAoInE"}
|
|
@@ -30,7 +30,7 @@ import { basename, extname, isAbsolute, join, relative, resolve, } from "node:pa
|
|
|
30
30
|
import { pathToFileURL } from "node:url";
|
|
31
31
|
import { resolveABMaxPlaygroundSessions } from "../ab.js";
|
|
32
32
|
import { httpChannel } from "../channels.js";
|
|
33
|
-
import { DEFAULT_MODEL, MAX_RUNNING_TURNS_LIMIT, toPublicCloudInfo, } from "../types.js";
|
|
33
|
+
import { cursorAccountServersHasMixedWildcard, DEFAULT_MODEL, MAX_RUNNING_TURNS_LIMIT, normalizeCursorAccountServers, toPublicCloudInfo, } from "../types.js";
|
|
34
34
|
import { registerAuthoredModuleLoaders } from "./authored-loaders.js";
|
|
35
35
|
import { normalizePositiveInt } from "./bounded-int.js";
|
|
36
36
|
import { getDefinitionKind } from "./brand.js";
|
|
@@ -238,10 +238,10 @@ export function formatDiagnostics(diagnostics) {
|
|
|
238
238
|
// ============================================================================
|
|
239
239
|
function loadAgent(ctx, agentDir, options) {
|
|
240
240
|
return __awaiter(this, void 0, void 0, function* () {
|
|
241
|
-
var _a;
|
|
242
|
-
var
|
|
241
|
+
var _a, _b, _c;
|
|
242
|
+
var _d;
|
|
243
243
|
const config = yield loadAgentConfig(ctx, agentDir, options.isRoot);
|
|
244
|
-
const name = (
|
|
244
|
+
const name = (_d = config === null || config === void 0 ? void 0 : config.name) !== null && _d !== void 0 ? _d : options.name;
|
|
245
245
|
const authoredInstructions = yield loadInstructions(ctx, agentDir, config, options.isRoot);
|
|
246
246
|
// Root agents without authored instructions stay undefined (diagnostic
|
|
247
247
|
// already recorded). Subagents still get the identity line alone.
|
|
@@ -267,10 +267,21 @@ function loadAgent(ctx, agentDir, options) {
|
|
|
267
267
|
// `send({ cloud })` merges it under cloudOverride for handoff sessions.
|
|
268
268
|
const cloud = config === null || config === void 0 ? void 0 : config.cloud;
|
|
269
269
|
const localCwd = resolveAgentCwd(ctx, (_a = config === null || config === void 0 ? void 0 : config.local) === null || _a === void 0 ? void 0 : _a.cwd);
|
|
270
|
-
const
|
|
270
|
+
const localWorkspaceDir = resolveAgentCwd(ctx, (_b = config === null || config === void 0 ? void 0 : config.local) === null || _b === void 0 ? void 0 : _b.workspaceDir);
|
|
271
|
+
const localSandbox = ((_c = config === null || config === void 0 ? void 0 : config.local) === null || _c === void 0 ? void 0 : _c.sandbox) === true;
|
|
272
|
+
if (localCwd !== undefined && localWorkspaceDir !== undefined) {
|
|
273
|
+
ctx.diagnostics.push({
|
|
274
|
+
severity: "warning",
|
|
275
|
+
path: displayPath(ctx, join(agentDir, "agent.ts")),
|
|
276
|
+
message: "local.workspaceDir pins every session to one directory, so local.cwd is unused. Drop one of them.",
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
const local = localCwd === undefined && localWorkspaceDir === undefined && !localSandbox
|
|
271
280
|
? undefined
|
|
272
281
|
: {
|
|
273
282
|
cwd: localCwd,
|
|
283
|
+
workspaceDir: localWorkspaceDir,
|
|
284
|
+
sandbox: localSandbox,
|
|
274
285
|
};
|
|
275
286
|
const hosting = resolveHosting(ctx, agentDir, config === null || config === void 0 ? void 0 : config.hosting);
|
|
276
287
|
const concurrency = resolveConcurrency(ctx, agentDir, config === null || config === void 0 ? void 0 : config.concurrency);
|
|
@@ -710,9 +721,8 @@ function hasAuthorizationHeader(headers) {
|
|
|
710
721
|
}
|
|
711
722
|
function connectionTransport(connection) {
|
|
712
723
|
if ("cursorAccount" in connection && connection.cursorAccount === true) {
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
: { servers: connection.servers }));
|
|
724
|
+
const servers = normalizeCursorAccountServers(connection.servers);
|
|
725
|
+
return Object.assign({ cursorAccount: true }, (servers === undefined ? {} : { servers }));
|
|
716
726
|
}
|
|
717
727
|
if ("agent" in connection && !isBlank(connection.agent)) {
|
|
718
728
|
return { agent: connection.agent };
|
|
@@ -830,6 +840,16 @@ function loadConnections(ctx, connectionsDir) {
|
|
|
830
840
|
});
|
|
831
841
|
continue;
|
|
832
842
|
}
|
|
843
|
+
if ("cursorAccount" in connection &&
|
|
844
|
+
connection.cursorAccount === true &&
|
|
845
|
+
cursorAccountServersHasMixedWildcard(connection.servers)) {
|
|
846
|
+
ctx.diagnostics.push({
|
|
847
|
+
severity: "error",
|
|
848
|
+
path: displayPath(ctx, entryPath),
|
|
849
|
+
message: 'Cursor account MCP connection `servers` cannot mix `"*"` with connector names. Use `"*"` / `["*"]` (or omit `servers`) for every connected server, or list only concrete connector names.',
|
|
850
|
+
});
|
|
851
|
+
continue;
|
|
852
|
+
}
|
|
833
853
|
const transport = connectionTransport(connection);
|
|
834
854
|
if (transport === undefined) {
|
|
835
855
|
ctx.diagnostics.push({
|
|
@@ -846,7 +866,7 @@ function loadConnections(ctx, connectionsDir) {
|
|
|
846
866
|
ctx.diagnostics.push({
|
|
847
867
|
severity: "error",
|
|
848
868
|
path: displayPath(ctx, entryPath),
|
|
849
|
-
message:
|
|
869
|
+
message: 'Cursor account MCP connection `servers` must be a non-empty list of connector names, `"*"` / `["*"]` for every connected server, or omitted for the same all-connectors behavior.',
|
|
850
870
|
});
|
|
851
871
|
continue;
|
|
852
872
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"distribution.d.ts","sourceRoot":"","sources":["../../src/internal/distribution.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,qEAAqE;AACrE,eAAO,MAAM,YAAY,iBAAiB,CAAC;AAE3C,6EAA6E;AAC7E,eAAO,MAAM,mBAAmB,2BAA2B,CAAC;AAE5D;;;GAGG;AACH,eAAO,MAAM,qBAAqB,qBAAqB,CAAC;AAExD;;;GAGG;AACH,eAAO,MAAM,WAAW,aAAa,CAAC;AACtC,eAAO,MAAM,UAAU,EAAE,MAA2B,CAAC;AACrD,eAAO,MAAM,gBAAgB,EAAE,MAAiC,CAAC;AAEjE;;;GAGG;AACH,eAAO,MAAM,gBAAgB,EAAE,MAA+B,CAAC;AAc/D;;;;;GAKG;AACH,wBAAgB,sBAAsB,IAAI,OAAO,CAGhD;AAED;;;GAGG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED,kEAAkE;AAClE,wBAAgB,cAAc,IAAI,MAAM,CAOvC;AAED;;;;;;GAMG;AACH,eAAO,MAAM,qBAAqB,EAAE,aAAa,CAC/C,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"distribution.d.ts","sourceRoot":"","sources":["../../src/internal/distribution.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,qEAAqE;AACrE,eAAO,MAAM,YAAY,iBAAiB,CAAC;AAE3C,6EAA6E;AAC7E,eAAO,MAAM,mBAAmB,2BAA2B,CAAC;AAE5D;;;GAGG;AACH,eAAO,MAAM,qBAAqB,qBAAqB,CAAC;AAExD;;;GAGG;AACH,eAAO,MAAM,WAAW,aAAa,CAAC;AACtC,eAAO,MAAM,UAAU,EAAE,MAA2B,CAAC;AACrD,eAAO,MAAM,gBAAgB,EAAE,MAAiC,CAAC;AAEjE;;;GAGG;AACH,eAAO,MAAM,gBAAgB,EAAE,MAA+B,CAAC;AAc/D;;;;;GAKG;AACH,wBAAgB,sBAAsB,IAAI,OAAO,CAGhD;AAED;;;GAGG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAED,kEAAkE;AAClE,wBAAgB,cAAc,IAAI,MAAM,CAOvC;AAED;;;;;;GAMG;AACH,eAAO,MAAM,qBAAqB,EAAE,aAAa,CAC/C,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAqBzC,CAAC"}
|
|
@@ -86,6 +86,8 @@ export const AUTHORING_ENTRY_FILES = [
|
|
|
86
86
|
["hooks", "hooks.js"],
|
|
87
87
|
["prompt", "prompt.js"],
|
|
88
88
|
["evals", "evals.js"],
|
|
89
|
+
["evals/reporters", "evals/reporters.js"],
|
|
90
|
+
["evals/loaders", "evals/loaders.js"],
|
|
89
91
|
["ab", "ab.js"],
|
|
90
92
|
["storage", "storage.js"],
|
|
91
93
|
["storage/file-kv", "storage-backends/file-kv.js"],
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model call behind `t.judge.*`.
|
|
3
|
+
*
|
|
4
|
+
* Grading is completion-only in intent: the judge reads a prompt and returns a
|
|
5
|
+
* choice. It is nonetheless a real SDK session, and its prompt embeds output
|
|
6
|
+
* from the agent under test — attacker-controlled text in exactly the way a
|
|
7
|
+
* prompt injection needs. A reply that talks the judge into running a tool would
|
|
8
|
+
* do so on the machine running the eval CLI, under the runner's credentials.
|
|
9
|
+
*
|
|
10
|
+
* The session is therefore confined on every axis the SDK exposes:
|
|
11
|
+
*
|
|
12
|
+
* - `sandboxOptions.enabled` so any tool call the judge is talked into is
|
|
13
|
+
* confined rather than run directly against the host.
|
|
14
|
+
* - `mcpServers: {}` so no MCP surface is reachable.
|
|
15
|
+
* - `settingSources: []` plus an empty temp `cwd`, so there is no repo to act on
|
|
16
|
+
* and no ambient `AGENTS.md` / `.cursor` rule can steer a verdict.
|
|
17
|
+
*
|
|
18
|
+
* The prompt side is hardened separately: `evals/judge.ts` fences untrusted
|
|
19
|
+
* content behind a per-call boundary. Both halves matter — this one limits blast
|
|
20
|
+
* radius, that one removes the instruction channel.
|
|
21
|
+
*
|
|
22
|
+
* Grading also declines outright when the credential's backend and the SDK's
|
|
23
|
+
* backend disagree, rather than sending a key somewhere it was not minted for
|
|
24
|
+
* (see {@link assertJudgeBackendMatch}).
|
|
25
|
+
*/
|
|
26
|
+
import type { EvalJudgeCaller } from "../evals/judge.js";
|
|
27
|
+
import type { ModelSetting } from "../types.js";
|
|
28
|
+
/** Env override for the default judge model (below `evals.config.ts`). */
|
|
29
|
+
export declare const EVAL_JUDGE_MODEL_ENV = "AGENTKIT_EVAL_JUDGE_MODEL";
|
|
30
|
+
/**
|
|
31
|
+
* Refuse to grade when the judge credential belongs to a different backend than
|
|
32
|
+
* the SDK will send it to.
|
|
33
|
+
*
|
|
34
|
+
* `resolveApiKey` resolves against `AGENT_SERVE_CURSOR_BACKEND_URL`, which the
|
|
35
|
+
* SDK does not read. When only that override is set, the key and the judge
|
|
36
|
+
* prompt — which embeds agent output — would go to the default host instead of
|
|
37
|
+
* the intended one. There is no backend field on `AgentOptions` to reconcile
|
|
38
|
+
* them, so the honest move is to decline rather than send a credential
|
|
39
|
+
* somewhere it was not minted for.
|
|
40
|
+
*/
|
|
41
|
+
export declare function assertJudgeBackendMatch(credentialBackendUrl: string, sdkBackend?: string): void;
|
|
42
|
+
/**
|
|
43
|
+
* Resolve the judge model, innermost first: per-call → per-eval →
|
|
44
|
+
* `evals.config.ts` → {@link EVAL_JUDGE_MODEL_ENV}.
|
|
45
|
+
*/
|
|
46
|
+
export declare function resolveJudgeModel(layers: {
|
|
47
|
+
call?: ModelSetting;
|
|
48
|
+
evalLevel?: ModelSetting;
|
|
49
|
+
config?: ModelSetting;
|
|
50
|
+
env?: string | undefined;
|
|
51
|
+
}): ModelSetting | undefined;
|
|
52
|
+
/**
|
|
53
|
+
* Ask the judge model for a grade.
|
|
54
|
+
*
|
|
55
|
+
* Throws {@link EvalJudgeUnavailableError} when no credentials are available so
|
|
56
|
+
* the runner can skip the eval visibly instead of reporting a spurious failure.
|
|
57
|
+
*/
|
|
58
|
+
export declare const callJudgeModel: EvalJudgeCaller;
|
|
59
|
+
//# sourceMappingURL=eval-judge-model.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-judge-model.d.ts","sourceRoot":"","sources":["../../src/internal/eval-judge-model.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAKH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAOhD,0EAA0E;AAC1E,eAAO,MAAM,oBAAoB,8BAA8B,CAAC;AAsBhE;;;;;;;;;;GAUG;AACH,wBAAgB,uBAAuB,CACrC,oBAAoB,EAAE,MAAM,EAC5B,UAAU,GAAE,MAAwB,GACnC,IAAI,CASN;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE;IACxC,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,SAAS,CAAC,EAAE,YAAY,CAAC;IACzB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1B,GAAG,YAAY,GAAG,SAAS,CAQ3B;AAED;;;;;GAKG;AACH,eAAO,MAAM,cAAc,EAAE,eAoC5B,CAAC"}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model call behind `t.judge.*`.
|
|
3
|
+
*
|
|
4
|
+
* Grading is completion-only in intent: the judge reads a prompt and returns a
|
|
5
|
+
* choice. It is nonetheless a real SDK session, and its prompt embeds output
|
|
6
|
+
* from the agent under test — attacker-controlled text in exactly the way a
|
|
7
|
+
* prompt injection needs. A reply that talks the judge into running a tool would
|
|
8
|
+
* do so on the machine running the eval CLI, under the runner's credentials.
|
|
9
|
+
*
|
|
10
|
+
* The session is therefore confined on every axis the SDK exposes:
|
|
11
|
+
*
|
|
12
|
+
* - `sandboxOptions.enabled` so any tool call the judge is talked into is
|
|
13
|
+
* confined rather than run directly against the host.
|
|
14
|
+
* - `mcpServers: {}` so no MCP surface is reachable.
|
|
15
|
+
* - `settingSources: []` plus an empty temp `cwd`, so there is no repo to act on
|
|
16
|
+
* and no ambient `AGENTS.md` / `.cursor` rule can steer a verdict.
|
|
17
|
+
*
|
|
18
|
+
* The prompt side is hardened separately: `evals/judge.ts` fences untrusted
|
|
19
|
+
* content behind a per-call boundary. Both halves matter — this one limits blast
|
|
20
|
+
* radius, that one removes the instruction channel.
|
|
21
|
+
*
|
|
22
|
+
* Grading also declines outright when the credential's backend and the SDK's
|
|
23
|
+
* backend disagree, rather than sending a key somewhere it was not minted for
|
|
24
|
+
* (see {@link assertJudgeBackendMatch}).
|
|
25
|
+
*/
|
|
26
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
27
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
28
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
29
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
30
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
31
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
32
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
33
|
+
});
|
|
34
|
+
};
|
|
35
|
+
import { mkdtemp, rm } from "node:fs/promises";
|
|
36
|
+
import { tmpdir } from "node:os";
|
|
37
|
+
import { join } from "node:path";
|
|
38
|
+
import { EvalJudgeUnavailableError } from "../evals/judge.js";
|
|
39
|
+
import { DEFAULT_CURSOR_BACKEND_URL, resolveApiKey, SIGN_IN_HINT, } from "./cursor/credentials.js";
|
|
40
|
+
/** Env override for the default judge model (below `evals.config.ts`). */
|
|
41
|
+
export const EVAL_JUDGE_MODEL_ENV = "AGENTKIT_EVAL_JUDGE_MODEL";
|
|
42
|
+
/**
|
|
43
|
+
* Backend the Cursor SDK will actually talk to.
|
|
44
|
+
*
|
|
45
|
+
* `Agent.create` takes a credential but no backend, so the SDK picks its own
|
|
46
|
+
* host from `CURSOR_API_BASE_URL`. Kept next to {@link assertJudgeBackendMatch}
|
|
47
|
+
* so the two stay in step if the SDK's resolution changes.
|
|
48
|
+
*/
|
|
49
|
+
function sdkBackendUrl() {
|
|
50
|
+
var _a;
|
|
51
|
+
const configured = (_a = process.env.CURSOR_API_BASE_URL) === null || _a === void 0 ? void 0 : _a.trim();
|
|
52
|
+
return trimTrailingSlash(configured === undefined || configured === ""
|
|
53
|
+
? DEFAULT_CURSOR_BACKEND_URL
|
|
54
|
+
: configured);
|
|
55
|
+
}
|
|
56
|
+
function trimTrailingSlash(url) {
|
|
57
|
+
return url.replace(/\/+$/, "");
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Refuse to grade when the judge credential belongs to a different backend than
|
|
61
|
+
* the SDK will send it to.
|
|
62
|
+
*
|
|
63
|
+
* `resolveApiKey` resolves against `AGENT_SERVE_CURSOR_BACKEND_URL`, which the
|
|
64
|
+
* SDK does not read. When only that override is set, the key and the judge
|
|
65
|
+
* prompt — which embeds agent output — would go to the default host instead of
|
|
66
|
+
* the intended one. There is no backend field on `AgentOptions` to reconcile
|
|
67
|
+
* them, so the honest move is to decline rather than send a credential
|
|
68
|
+
* somewhere it was not minted for.
|
|
69
|
+
*/
|
|
70
|
+
export function assertJudgeBackendMatch(credentialBackendUrl, sdkBackend = sdkBackendUrl()) {
|
|
71
|
+
const credentialBackend = trimTrailingSlash(credentialBackendUrl);
|
|
72
|
+
if (credentialBackend === sdkBackend) {
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
throw new EvalJudgeUnavailableError(`eval judge credential is scoped to ${credentialBackend}, but the Cursor SDK would send it to ${sdkBackend}. ` +
|
|
76
|
+
"Set CURSOR_API_BASE_URL to the same backend (the SDK does not read AGENT_SERVE_CURSOR_BACKEND_URL).");
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Resolve the judge model, innermost first: per-call → per-eval →
|
|
80
|
+
* `evals.config.ts` → {@link EVAL_JUDGE_MODEL_ENV}.
|
|
81
|
+
*/
|
|
82
|
+
export function resolveJudgeModel(layers) {
|
|
83
|
+
var _a;
|
|
84
|
+
var _b, _c, _d;
|
|
85
|
+
const envModel = (_a = layers.env) === null || _a === void 0 ? void 0 : _a.trim();
|
|
86
|
+
return ((_d = (_c = (_b = layers.call) !== null && _b !== void 0 ? _b : layers.evalLevel) !== null && _c !== void 0 ? _c : layers.config) !== null && _d !== void 0 ? _d : (envModel === undefined || envModel === "" ? undefined : envModel));
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Ask the judge model for a grade.
|
|
90
|
+
*
|
|
91
|
+
* Throws {@link EvalJudgeUnavailableError} when no credentials are available so
|
|
92
|
+
* the runner can skip the eval visibly instead of reporting a spurious failure.
|
|
93
|
+
*/
|
|
94
|
+
export const callJudgeModel = (_a) => __awaiter(void 0, [_a], void 0, function* ({ prompt, model }) {
|
|
95
|
+
var _b;
|
|
96
|
+
var _c, _d, _e;
|
|
97
|
+
const resolved = yield resolveApiKey();
|
|
98
|
+
if (resolved === undefined) {
|
|
99
|
+
throw new EvalJudgeUnavailableError(`no Cursor credentials for the eval judge. ${SIGN_IN_HINT}`);
|
|
100
|
+
}
|
|
101
|
+
assertJudgeBackendMatch(resolved.backendUrl);
|
|
102
|
+
// Imported lazily so a tree of fully deterministic evals never loads the SDK.
|
|
103
|
+
const { Agent } = yield import("@cursor/sdk");
|
|
104
|
+
const cwd = yield mkdtemp(join(tmpdir(), "agentkit-eval-judge-"));
|
|
105
|
+
const agent = yield Agent.create({
|
|
106
|
+
apiKey: resolved.apiKey,
|
|
107
|
+
model: typeof model === "string" ? { id: model } : model,
|
|
108
|
+
// See the module comment: grading embeds untrusted agent output, so the
|
|
109
|
+
// session gets no MCP surface, no ambient settings, an empty working
|
|
110
|
+
// directory, and a sandbox around anything it does attempt.
|
|
111
|
+
mcpServers: {},
|
|
112
|
+
local: {
|
|
113
|
+
cwd,
|
|
114
|
+
settingSources: [],
|
|
115
|
+
sandboxOptions: { enabled: true },
|
|
116
|
+
},
|
|
117
|
+
});
|
|
118
|
+
try {
|
|
119
|
+
const run = yield agent.send(prompt);
|
|
120
|
+
const result = yield run.wait();
|
|
121
|
+
if (result.status === "error" || result.status === "cancelled") {
|
|
122
|
+
const detail = (_d = (_c = (_b = result.error) === null || _b === void 0 ? void 0 : _b.message) !== null && _c !== void 0 ? _c : result.result) !== null && _d !== void 0 ? _d : result.status;
|
|
123
|
+
throw new Error(`eval judge run failed: ${detail}`);
|
|
124
|
+
}
|
|
125
|
+
return (_e = result.result) !== null && _e !== void 0 ? _e : "";
|
|
126
|
+
}
|
|
127
|
+
finally {
|
|
128
|
+
agent.close();
|
|
129
|
+
yield rm(cwd, { recursive: true, force: true }).catch(() => undefined);
|
|
130
|
+
}
|
|
131
|
+
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-run-store.d.ts","sourceRoot":"","sources":["../../src/internal/eval-run-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAGL,KAAK,eAAe,EAErB,MAAM,aAAa,CAAC;AASrB,YAAY,EACV,eAAe,EACf,iBAAiB,IAAI,cAAc,EACnC,gBAAgB,EAChB,eAAe,GAChB,MAAM,aAAa,CAAC;AAErB;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI,CAAC;IACjC,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AASF,qBAAa,YAAY;IAUrB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;IAZ3B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsC;IAC3D,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsC;IACnE,OAAO,CAAC,cAAc,CAA4B;IAClD,OAAO,CAAC,QAAQ,CAAS;IACzB,wEAAwE;IACxE,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAEhC,YACmB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI;IAC/C,oEAAoE;IACnD,OAAO,CAAC,EAAE,cAAc,GAAG,SAAS,EACnD;IAEE,IAAI,IAAI,OAAO,CAAC;QACpB,KAAK,EAAE,KAAK,CAAC;YACX,EAAE,EAAE,MAAM,CAAC;YACX,MAAM,EAAE,MAAM,CAAC;YACf,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC,CAAC;QACH,MAAM,EAAE,cAAc,CAAC;KACxB,CAAC,CAcD;IAEK,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAG7D;IAED,+EAA+E;IACzE,QAAQ,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAG3C;IAED;;;;OAIG;IACG,kBAAkB,IAAI,OAAO,CAAC;QAClC,IAAI,EAAE,eAAe,EAAE,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,CAOD;IAED,cAAc,IAAI,MAAM,GAAG,SAAS,CAEnC;IAEK,KAAK,CAAC,KAAK,GAAE,iBAAsB,GAAG,OAAO,CAAC,eAAe,CAAC,
|
|
1
|
+
{"version":3,"file":"eval-run-store.d.ts","sourceRoot":"","sources":["../../src/internal/eval-run-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAGL,KAAK,eAAe,EAErB,MAAM,aAAa,CAAC;AASrB,YAAY,EACV,eAAe,EACf,iBAAiB,IAAI,cAAc,EACnC,gBAAgB,EAChB,eAAe,GAChB,MAAM,aAAa,CAAC;AAErB;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI,CAAC;IACjC,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AASF,qBAAa,YAAY;IAUrB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;IAZ3B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsC;IAC3D,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsC;IACnE,OAAO,CAAC,cAAc,CAA4B;IAClD,OAAO,CAAC,QAAQ,CAAS;IACzB,wEAAwE;IACxE,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAEhC,YACmB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI;IAC/C,oEAAoE;IACnD,OAAO,CAAC,EAAE,cAAc,GAAG,SAAS,EACnD;IAEE,IAAI,IAAI,OAAO,CAAC;QACpB,KAAK,EAAE,KAAK,CAAC;YACX,EAAE,EAAE,MAAM,CAAC;YACX,MAAM,EAAE,MAAM,CAAC;YACf,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC,CAAC;QACH,MAAM,EAAE,cAAc,CAAC;KACxB,CAAC,CAcD;IAEK,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAG7D;IAED,+EAA+E;IACzE,QAAQ,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAG3C;IAED;;;;OAIG;IACG,kBAAkB,IAAI,OAAO,CAAC;QAClC,IAAI,EAAE,eAAe,EAAE,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,CAOD;IAED,cAAc,IAAI,MAAM,GAAG,SAAS,CAEnC;IAEK,KAAK,CAAC,KAAK,GAAE,iBAAsB,GAAG,OAAO,CAAC,eAAe,CAAC,CAkEnE;IAED;;;OAGG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAqBpD;IAED,OAAO,CAAC,UAAU;IAMlB;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ7B;YAEa,kBAAkB;IAkDhC,OAAO,CAAC,YAAY;IAepB,6EAA6E;IAC7E,OAAO,CAAC,OAAO;IAIf;;;;;;OAMG;IACH,KAAK,IAAI,IAAI,CAcZ;YAEa,OAAO;IAoHrB,OAAO,CAAC,eAAe;CA8CxB;AAmFD,qBAAa,oBAAqB,SAAQ,KAAK;IACjC,QAAQ,CAAC,WAAW,EAAE,MAAM;IAAxC,YAAqB,WAAW,EAAE,MAAM,EAGvC;CACF;AAED,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,cAGC;CACF;AAED,qBAAa,mBAAoB,SAAQ,KAAK;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM;IAAlC,YAAqB,KAAK,EAAE,MAAM,EAGjC;CACF;AAED,qBAAa,sBAAuB,SAAQ,KAAK;IAE7C,QAAQ,CAAC,KAAK,EAAE,MAAM;IACtB,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC,QAAQ,CAAC;IAF5C,YACW,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,eAAe,CAAC,QAAQ,CAAC,EAI3C;CACF"}
|
|
@@ -104,6 +104,8 @@ export class EvalRunStore {
|
|
|
104
104
|
summary: {
|
|
105
105
|
passed: 0,
|
|
106
106
|
failed: 0,
|
|
107
|
+
scored: 0,
|
|
108
|
+
skipped: 0,
|
|
107
109
|
total: cases.length,
|
|
108
110
|
done: 0,
|
|
109
111
|
},
|
|
@@ -351,6 +353,7 @@ export class EvalRunStore {
|
|
|
351
353
|
});
|
|
352
354
|
}
|
|
353
355
|
applyCaseResult(runId, result) {
|
|
356
|
+
var _a, _b;
|
|
354
357
|
const run = this.runs.get(runId);
|
|
355
358
|
if (run === undefined) {
|
|
356
359
|
return;
|
|
@@ -362,6 +365,9 @@ export class EvalRunStore {
|
|
|
362
365
|
const alreadyCounted = c.status === "done";
|
|
363
366
|
c.status = "done";
|
|
364
367
|
c.ok = result.ok;
|
|
368
|
+
c.verdict = result.verdict;
|
|
369
|
+
c.skipReason = result.skipReason;
|
|
370
|
+
c.metadata = result.metadata;
|
|
365
371
|
c.assertions = result.assertions;
|
|
366
372
|
c.durationMs = result.durationMs;
|
|
367
373
|
c.logs = result.logs;
|
|
@@ -389,12 +395,8 @@ export class EvalRunStore {
|
|
|
389
395
|
return;
|
|
390
396
|
}
|
|
391
397
|
run.summary.done += 1;
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
}
|
|
395
|
-
else {
|
|
396
|
-
run.summary.failed += 1;
|
|
397
|
-
}
|
|
398
|
+
const verdict = (_a = result.verdict) !== null && _a !== void 0 ? _a : (result.ok ? "passed" : "failed");
|
|
399
|
+
run.summary[verdict] = ((_b = run.summary[verdict]) !== null && _b !== void 0 ? _b : 0) + 1;
|
|
398
400
|
}
|
|
399
401
|
}
|
|
400
402
|
function markInterruptedRun(run) {
|
|
@@ -426,22 +428,12 @@ function markCancelledRun(run) {
|
|
|
426
428
|
reconcileRunSummary(run);
|
|
427
429
|
}
|
|
428
430
|
function reconcileRunSummary(run) {
|
|
429
|
-
|
|
430
|
-
|
|
431
|
+
var _a;
|
|
432
|
+
const counts = { passed: 0, failed: 0, scored: 0, skipped: 0 };
|
|
431
433
|
for (const c of run.cases) {
|
|
432
|
-
|
|
433
|
-
passed += 1;
|
|
434
|
-
}
|
|
435
|
-
else {
|
|
436
|
-
failed += 1;
|
|
437
|
-
}
|
|
434
|
+
counts[(_a = c.verdict) !== null && _a !== void 0 ? _a : (c.ok === true ? "passed" : "failed")] += 1;
|
|
438
435
|
}
|
|
439
|
-
run.summary = {
|
|
440
|
-
passed,
|
|
441
|
-
failed,
|
|
442
|
-
total: run.cases.length,
|
|
443
|
-
done: run.cases.length,
|
|
444
|
-
};
|
|
436
|
+
run.summary = Object.assign(Object.assign({}, counts), { total: run.cases.length, done: run.cases.length });
|
|
445
437
|
}
|
|
446
438
|
function snapshotListConfig(config, durableRuns) {
|
|
447
439
|
return Object.assign(Object.assign(Object.assign({}, (config.maxConcurrency === undefined
|
|
@@ -7,7 +7,8 @@
|
|
|
7
7
|
* Drive with `t.send`, assert with `t.succeeded` / `t.calledTool` /
|
|
8
8
|
* `t.messageIncludes` / `t.check`.
|
|
9
9
|
*/
|
|
10
|
-
import type {
|
|
10
|
+
import type { EvalJudgeCaller } from "../evals/judge.js";
|
|
11
|
+
import type { EvalCaseDefinition, EvalConfigFile, EvalReporter, EvalReporterTarget, EvalRunResult } from "../evals.js";
|
|
11
12
|
import type { SessionEvent } from "../types.js";
|
|
12
13
|
export interface DiscoveredEval {
|
|
13
14
|
/** Runnable datapoint id (`weather/nyc` or file id for single-test files). */
|
|
@@ -17,30 +18,7 @@ export interface DiscoveredEval {
|
|
|
17
18
|
path: string;
|
|
18
19
|
definition: EvalCaseDefinition;
|
|
19
20
|
}
|
|
20
|
-
export
|
|
21
|
-
id: string;
|
|
22
|
-
path: string;
|
|
23
|
-
description?: string;
|
|
24
|
-
ok: boolean;
|
|
25
|
-
assertions: EvalAssertionResult[];
|
|
26
|
-
sessionId?: string;
|
|
27
|
-
/** Messages passed to `t.send` (eval inputs). */
|
|
28
|
-
inputs: string[];
|
|
29
|
-
finalText?: string;
|
|
30
|
-
tools?: string[];
|
|
31
|
-
/** Tool calls with args/results from the trajectory. */
|
|
32
|
-
toolCalls?: Array<{
|
|
33
|
-
toolName: string;
|
|
34
|
-
args?: unknown;
|
|
35
|
-
output?: unknown;
|
|
36
|
-
isError: boolean;
|
|
37
|
-
}>;
|
|
38
|
-
error?: string;
|
|
39
|
-
logs: string[];
|
|
40
|
-
/** Structured scores from {@link import("../evals.js").EvalTestContext.metric}. */
|
|
41
|
-
metrics: Record<string, string | number>;
|
|
42
|
-
durationMs: number;
|
|
43
|
-
}
|
|
21
|
+
export type { EvalRunResult } from "../evals.js";
|
|
44
22
|
export interface EvalSelectionOptions {
|
|
45
23
|
filterIds?: string[];
|
|
46
24
|
tags?: string[];
|
|
@@ -49,6 +27,21 @@ export interface RunEvalsOptions extends EvalSelectionOptions {
|
|
|
49
27
|
projectRoot: string;
|
|
50
28
|
baseUrl: string;
|
|
51
29
|
timeoutMs?: number;
|
|
30
|
+
/** Override `evals.config.ts` `maxConcurrency` (CLI `--max-concurrency`). */
|
|
31
|
+
maxConcurrency?: number;
|
|
32
|
+
/**
|
|
33
|
+
* Reporters attached by the caller (CLI `--junit` / `--artifacts`), in
|
|
34
|
+
* addition to the ones declared in `evals.config.ts` and on each eval.
|
|
35
|
+
*/
|
|
36
|
+
reporters?: readonly EvalReporter[];
|
|
37
|
+
/** Ignore reporters declared in config and eval files (`--skip-report`). */
|
|
38
|
+
skipReport?: boolean;
|
|
39
|
+
/** Reported to reporters so a destination can label the run. */
|
|
40
|
+
target?: EvalReporterTarget;
|
|
41
|
+
/** Treat soft threshold misses as failures when summarizing. */
|
|
42
|
+
strict?: boolean;
|
|
43
|
+
/** Injected judge model call; defaults to the Cursor SDK caller. */
|
|
44
|
+
judgeCaller?: EvalJudgeCaller;
|
|
52
45
|
bearerToken?: string;
|
|
53
46
|
/** Extra request headers (e.g. hosted engineAccess). */
|
|
54
47
|
headers?: Record<string, string>;
|
|
@@ -76,6 +69,19 @@ export declare function discoverEvals(projectRoot: string, importModule?: (absol
|
|
|
76
69
|
evals: DiscoveredEval[];
|
|
77
70
|
config: Partial<EvalConfigFile>;
|
|
78
71
|
}>;
|
|
72
|
+
/**
|
|
73
|
+
* Expand one eval module's default export into runnable datapoints.
|
|
74
|
+
*
|
|
75
|
+
* A file normally exports a single `defineEval(...)`. It may instead export an
|
|
76
|
+
* **array** of them to fan out over a dataset, in which case ids come from the
|
|
77
|
+
* file name plus a zero-padded index in array order (`sql/0000`, `sql/0001`, …).
|
|
78
|
+
* Prefer `cases` when the datapoints are hand-written and deserve stable names;
|
|
79
|
+
* the array form is for generated datasets where the index *is* the name.
|
|
80
|
+
*/
|
|
81
|
+
export declare function expandEvalModule(fileId: string, defaultExport: unknown): Array<{
|
|
82
|
+
id: string;
|
|
83
|
+
definition: EvalCaseDefinition;
|
|
84
|
+
}>;
|
|
79
85
|
/** Apply id/tag filters used by CLI, playground, and `runDiscoveredEvals`. */
|
|
80
86
|
export declare function filterDiscoveredEvals(evals: readonly DiscoveredEval[], options?: EvalSelectionOptions): DiscoveredEval[];
|
|
81
87
|
export declare function runDiscoveredEvals(options: RunEvalsOptions): Promise<EvalRunResult[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-runner.d.ts","sourceRoot":"","sources":["../../src/internal/eval-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;
|
|
1
|
+
{"version":3,"file":"eval-runner.d.ts","sourceRoot":"","sources":["../../src/internal/eval-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAkBH,OAAO,KAAK,EAEV,eAAe,EAIhB,MAAM,mBAAmB,CAAC;AAS3B,OAAO,KAAK,EACV,kBAAkB,EAClB,cAAc,EAEd,YAAY,EACZ,kBAAkB,EAClB,aAAa,EAOd,MAAM,aAAa,CAAC;AASrB,OAAO,KAAK,EAAgB,YAAY,EAAE,MAAM,aAAa,CAAC;AAW9D,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,EAAE,EAAE,MAAM,CAAC;IACX,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,kBAAkB,CAAC;CAChC;AAED,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,MAAM,WAAW,oBAAoB;IACnC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,eAAgB,SAAQ,oBAAoB;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,4EAA4E;IAC5E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,gEAAgE;IAChE,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,gEAAgE;IAChE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,oEAAoE;IACpE,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IACxC,4EAA4E;IAC5E,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,iDAAiD;IACjD,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,KAAK,IAAI,CAAC;IAC7C,kFAAkF;IAClF,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IACxD,YAAY,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1E;;;OAGG;IACH,UAAU,CAAC,EAAE;QACX,KAAK,EAAE,SAAS,cAAc,EAAE,CAAC;QACjC,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;KACjC,CAAC;IACF,wEAAwE;IACxE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAuCD,wBAAsB,aAAa,CACjC,WAAW,EAAE,MAAM,EACnB,YAAY,GAAE,CACZ,YAAY,EAAE,MAAM,KACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAiB,GACpD,OAAO,CAAC;IACT,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;CACjC,CAAC,CA0CD;AAED;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,OAAO,GACrB,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAA;CAAE,CAAC,CAyBvD;AAED,8EAA8E;AAC9E,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,SAAS,cAAc,EAAE,EAChC,OAAO,GAAE,oBAAyB,GACjC,cAAc,EAAE,CAgBlB;AAED,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,aAAa,EAAE,CAAC,CAqF1B;AAwED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,cAAc,GAAG,MAAM,CAWvE;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,SAAS,cAAc,EAAE,GAC/B,cAAc,EAAE,EAAE,CAmBpB"}
|