@cursor/july 0.1.108 → 0.1.109
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -6
- package/README.md +3 -6
- package/dist/channels/bitbucket/api.d.ts +41 -0
- package/dist/channels/bitbucket/api.d.ts.map +1 -1
- package/dist/channels/bitbucket/api.js +260 -0
- package/dist/channels/bitbucket/binding.d.ts +4 -0
- package/dist/channels/bitbucket/binding.d.ts.map +1 -1
- package/dist/channels/bitbucket/binding.js +16 -0
- package/dist/channels/bitbucket/index.d.ts +1 -1
- package/dist/channels/bitbucket/index.d.ts.map +1 -1
- package/dist/channels/bitbucket/index.js +1 -1
- package/dist/channels/gitlab/api.d.ts +27 -0
- package/dist/channels/gitlab/api.d.ts.map +1 -1
- package/dist/channels/gitlab/api.js +88 -0
- package/dist/channels/gitlab/binding.d.ts +5 -0
- package/dist/channels/gitlab/binding.d.ts.map +1 -1
- package/dist/channels/gitlab/binding.js +10 -0
- package/dist/channels/gitlab/index.d.ts +1 -1
- package/dist/channels/gitlab/index.d.ts.map +1 -1
- package/dist/channels/gitlab/index.js +1 -1
- package/dist/channels/slack/dispatch.d.ts +10 -0
- package/dist/channels/slack/dispatch.d.ts.map +1 -1
- package/dist/channels/slack/dispatch.js +20 -3
- package/dist/channels/slack/slack-channel.d.ts +12 -5
- package/dist/channels/slack/slack-channel.d.ts.map +1 -1
- package/dist/channels/slack/slack-channel.js +59 -8
- package/dist/docs/404.html +2 -2
- package/dist/docs/assets/{app.BASKM3M4.js → app.Cr-wVbnB.js} +1 -1
- package/dist/docs/assets/{building-with-agents.md.CUSWxlP_.js → building-with-agents.md.D0KbSkJn.js} +2 -2
- package/dist/docs/assets/{building-with-agents.md.CUSWxlP_.lean.js → building-with-agents.md.D0KbSkJn.lean.js} +1 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.CFVQ4S17.js +1 -0
- package/dist/docs/assets/chunks/{VPLocalSearchBox.BOKxlYGP.js → VPLocalSearchBox.CVQERt56.js} +1 -1
- package/dist/docs/assets/chunks/{theme.DbDZW-zb.js → theme.Dnsd3XOn.js} +2 -2
- package/dist/docs/assets/concepts.md.B4o63Gul.js +1 -0
- package/dist/docs/assets/evals.md.C7JLjoEP.js +211 -0
- package/dist/docs/assets/evals.md.C7JLjoEP.lean.js +1 -0
- package/dist/docs/assets/guides_bitbucket.md.DZPzmUjU.js +10 -0
- package/dist/docs/assets/guides_bitbucket.md.DZPzmUjU.lean.js +1 -0
- package/dist/docs/assets/{guides_github.md.BH33UEBJ.js → guides_github.md.TZaTZlfz.js} +13 -3
- package/dist/docs/assets/{guides_github.md.BH33UEBJ.lean.js → guides_github.md.TZaTZlfz.lean.js} +1 -1
- package/dist/docs/assets/guides_gitlab.md.BmqwQfdG.js +14 -0
- package/dist/docs/assets/guides_gitlab.md.BmqwQfdG.lean.js +1 -0
- package/dist/docs/assets/{guides_webhooks.md.DKdA43Qm.js → guides_webhooks.md.CJK484ex.js} +2 -2
- package/dist/docs/assets/{hillclimbing.md.CpTGTCle.js → hillclimbing.md.BOiVo1tf.js} +1 -1
- package/dist/docs/assets/index.md.DD9Q2XuJ.js +5 -0
- package/dist/docs/assets/{index.md.BjH1w2ZW.lean.js → index.md.DD9Q2XuJ.lean.js} +1 -1
- package/dist/docs/assets/{reference_channels.md.D-qTqwcq.js → reference_channels.md.CAo-iK4j.js} +2 -2
- package/dist/docs/assets/{reference_channels.md.D-qTqwcq.lean.js → reference_channels.md.CAo-iK4j.lean.js} +1 -1
- package/dist/docs/assets/{reference_cli.md.Ca26u0Es.js → reference_cli.md.Deg7849l.js} +1 -1
- package/dist/docs/assets/{reference_extensions.md.CPWt00ds.js → reference_extensions.md.ZAVUyuEX.js} +2 -2
- package/dist/docs/assets/{reference_hooks.md.Ddt5DdgJ.js → reference_hooks.md.BlM_bOg6.js} +3 -3
- package/dist/docs/assets/{reference_hooks.md.Ddt5DdgJ.lean.js → reference_hooks.md.BlM_bOg6.lean.js} +1 -1
- package/dist/docs/assets/{reference_http-api.md.oySXBO8o.js → reference_http-api.md.BwaCo-VO.js} +1 -1
- package/dist/docs/assets/{reference_playground.md.4myJPxrf.js → reference_playground.md.DLnoaczX.js} +1 -1
- package/dist/docs/assets/{reference_playground.md.4myJPxrf.lean.js → reference_playground.md.DLnoaczX.lean.js} +1 -1
- package/dist/docs/assets/{reference_project-layout.md.DuBu9a96.js → reference_project-layout.md.BEU8MtQV.js} +3 -3
- package/dist/docs/assets/{reference_project-layout.md.DuBu9a96.lean.js → reference_project-layout.md.BEU8MtQV.lean.js} +1 -1
- package/dist/docs/assets/reference_sessions.md.CyXV1MUw.js +1 -0
- package/dist/docs/assets/skills_deploy.md.CWqi_ZxW.js +35 -0
- package/dist/docs/assets/skills_deploy.md.CWqi_ZxW.lean.js +1 -0
- package/dist/docs/assets/{skills_evals.md.BhovOvrl.js → skills_evals.md.DFxYPErF.js} +3 -3
- package/dist/docs/assets/skills_evals.md.DFxYPErF.lean.js +1 -0
- package/dist/docs/assets/{skills_framework-map.md.D-tFZhFS.js → skills_framework-map.md.BxLSOhSY.js} +1 -1
- package/dist/docs/assets/skills_index.md.DL7EHaQ-.js +1 -0
- package/dist/docs/assets/skills_index.md.DL7EHaQ-.lean.js +1 -0
- package/dist/docs/assets/{storage.md.BOHeqk2M.js → storage.md.BUrhJ-Zz.js} +4 -4
- package/dist/docs/assets/{storage.md.BOHeqk2M.lean.js → storage.md.BUrhJ-Zz.lean.js} +1 -1
- package/dist/docs/building-with-agents.html +5 -5
- package/dist/docs/building-with-agents.md +3 -2
- package/dist/docs/concepts.html +5 -5
- package/dist/docs/concepts.md +2 -4
- package/dist/docs/deployment.html +4 -4
- package/dist/docs/evals.html +161 -35
- package/dist/docs/evals.md +612 -297
- package/dist/docs/guides/agent-to-agent.html +4 -4
- package/dist/docs/guides/bitbucket.html +36 -0
- package/dist/docs/guides/bitbucket.md +84 -0
- package/dist/docs/guides/cloud-agents.html +4 -4
- package/dist/docs/guides/convert-automation.html +4 -4
- package/dist/docs/guides/github.html +17 -7
- package/dist/docs/guides/github.md +31 -0
- package/dist/docs/guides/gitlab.html +40 -0
- package/dist/docs/guides/gitlab.md +92 -0
- package/dist/docs/guides/grokbot-agents.html +4 -4
- package/dist/docs/guides/human-in-the-loop.html +4 -4
- package/dist/docs/guides/improve.html +4 -4
- package/dist/docs/guides/mcp-oauth.html +4 -4
- package/dist/docs/guides/opentelemetry.html +4 -4
- package/dist/docs/guides/slack.html +5 -5
- package/dist/docs/guides/webhooks.html +6 -6
- package/dist/docs/guides/webhooks.md +4 -2
- package/dist/docs/hashmap.json +1 -1
- package/dist/docs/hillclimbing.html +6 -6
- package/dist/docs/hillclimbing.md +2 -2
- package/dist/docs/index.html +6 -6
- package/dist/docs/index.md +10 -6
- package/dist/docs/llms-full.txt +1113 -778
- package/dist/docs/llms.txt +6 -5
- package/dist/docs/quickstart.html +4 -4
- package/dist/docs/reference/agent-config.html +4 -4
- package/dist/docs/reference/artifacts.html +4 -4
- package/dist/docs/reference/channels.html +6 -6
- package/dist/docs/reference/channels.md +16 -3
- package/dist/docs/reference/cli.html +6 -6
- package/dist/docs/reference/cli.md +6 -4
- package/dist/docs/reference/connections.html +4 -4
- package/dist/docs/reference/extensions.html +7 -7
- package/dist/docs/reference/extensions.md +0 -3
- package/dist/docs/reference/hooks.html +7 -7
- package/dist/docs/reference/hooks.md +9 -12
- package/dist/docs/reference/http-api.html +6 -6
- package/dist/docs/reference/http-api.md +2 -3
- package/dist/docs/reference/instructions.html +4 -4
- package/dist/docs/reference/playground.html +5 -5
- package/dist/docs/reference/playground.md +0 -4
- package/dist/docs/reference/project-layout.html +7 -7
- package/dist/docs/reference/project-layout.md +1 -8
- package/dist/docs/reference/prompt.html +4 -4
- package/dist/docs/reference/result.html +4 -4
- package/dist/docs/reference/schedules.html +4 -4
- package/dist/docs/reference/sessions.html +5 -5
- package/dist/docs/reference/sessions.md +1 -2
- package/dist/docs/reference/skills.html +4 -4
- package/dist/docs/reference/subagents.html +4 -4
- package/dist/docs/reference/tools.html +4 -4
- package/dist/docs/scaffolding-agents.html +4 -4
- package/dist/docs/skills/create-agent.html +4 -4
- package/dist/docs/skills/debug.html +4 -4
- package/dist/docs/skills/deploy.html +61 -0
- package/dist/docs/skills/deploy.md +161 -0
- package/dist/docs/skills/evals.html +8 -8
- package/dist/docs/skills/evals.md +44 -6
- package/dist/docs/skills/framework-map.html +5 -5
- package/dist/docs/skills/framework-map.md +1 -2
- package/dist/docs/skills/github.html +4 -4
- package/dist/docs/skills/hillclimb.html +4 -4
- package/dist/docs/skills/index.html +6 -6
- package/dist/docs/skills/index.md +1 -1
- package/dist/docs/skills/mcp-auth.html +4 -4
- package/dist/docs/skills/otel.html +4 -4
- package/dist/docs/skills/setup-slack.html +4 -4
- package/dist/docs/storage.html +8 -8
- package/dist/docs/storage.md +15 -24
- package/dist/docs/templates/agentic-owners.html +4 -4
- package/dist/docs/templates/agents-md.html +4 -4
- package/dist/docs/templates/code-wiki.html +4 -4
- package/dist/docs/templates/demo.html +4 -4
- package/dist/docs/templates/grokbot-agents.html +4 -4
- package/dist/docs/templates/pr-autofixer.html +4 -4
- package/dist/docs/templates/security-help.html +4 -4
- package/dist/docs/templates/security-reviewer.html +4 -4
- package/dist/docs/templates/triage.html +4 -4
- package/dist/docs/troubleshooting.html +4 -4
- package/dist/extensions.d.ts +2 -3
- package/dist/extensions.d.ts.map +1 -1
- package/dist/extensions.js +2 -5
- package/dist/index.d.ts +2 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -2
- package/dist/internal/authored-alias-hooks.d.ts +5 -0
- package/dist/internal/authored-alias-hooks.d.ts.map +1 -1
- package/dist/internal/authored-alias-hooks.js +17 -0
- package/dist/internal/authored-loaders.d.ts +4 -0
- package/dist/internal/authored-loaders.d.ts.map +1 -1
- package/dist/internal/authored-loaders.js +21 -2
- package/dist/internal/cli-ax.d.ts +1 -2
- package/dist/internal/cli-ax.d.ts.map +1 -1
- package/dist/internal/cli-ax.js +1 -2
- package/dist/internal/continuation-channel.d.ts.map +1 -1
- package/dist/internal/continuation-channel.js +2 -2
- package/dist/internal/continuation-identity.js +8 -3
- package/dist/internal/discovery/agent.d.ts +1 -1
- package/dist/internal/discovery/agent.d.ts.map +1 -1
- package/dist/internal/discovery/agent.js +0 -5
- package/dist/internal/discovery/extension-overlay.d.ts +1 -2
- package/dist/internal/discovery/extension-overlay.d.ts.map +1 -1
- package/dist/internal/discovery/extension-overlay.js +0 -14
- package/dist/internal/discovery/extensions.d.ts +1 -2
- package/dist/internal/discovery/extensions.d.ts.map +1 -1
- package/dist/internal/discovery/extensions.js +4 -22
- package/dist/internal/discovery/info.d.ts.map +1 -1
- package/dist/internal/discovery/info.js +42 -24
- package/dist/internal/discovery/modules.js +0 -1
- package/dist/internal/discovery/project.d.ts.map +1 -1
- package/dist/internal/discovery/project.js +0 -16
- package/dist/internal/eval-runner.js +0 -1
- package/dist/internal/framework-file-storage.d.ts +4 -5
- package/dist/internal/framework-file-storage.d.ts.map +1 -1
- package/dist/internal/framework-file-storage.js +4 -5
- package/dist/internal/framework-storage-selection.d.ts +2 -2
- package/dist/internal/framework-storage-selection.js +2 -2
- package/dist/internal/init-scaffold.d.ts.map +1 -1
- package/dist/internal/init-scaffold.js +1 -2
- package/dist/internal/install-cursor-skills.d.ts +5 -2
- package/dist/internal/install-cursor-skills.d.ts.map +1 -1
- package/dist/internal/install-cursor-skills.js +25 -5
- package/dist/internal/run-client.d.ts +1 -1
- package/dist/internal/sdk-runner.js +1 -1
- package/dist/internal/server.js +0 -14
- package/dist/internal/session-engine.d.ts +5 -35
- package/dist/internal/session-engine.d.ts.map +1 -1
- package/dist/internal/session-engine.js +17 -162
- package/dist/internal/storage-coordinator.d.ts +5 -19
- package/dist/internal/storage-coordinator.d.ts.map +1 -1
- package/dist/internal/storage-coordinator.js +3 -62
- package/dist/internal/storage-roles.d.ts +4 -9
- package/dist/internal/storage-roles.d.ts.map +1 -1
- package/dist/internal/storage-roles.js +2 -2
- package/dist/playground/assets/index-Bhxzrcf6.css +1 -0
- package/dist/playground/assets/index-CqLX5uF3.js +67 -0
- package/dist/playground/index.html +2 -2
- package/dist/storage-backends/cursor-hosted-v2.d.ts +4 -5
- package/dist/storage-backends/cursor-hosted-v2.d.ts.map +1 -1
- package/dist/storage-backends/cursor-hosted-v2.js +4 -5
- package/dist/storage-backends/cursor-hosted.d.ts +1 -2
- package/dist/storage-backends/cursor-hosted.d.ts.map +1 -1
- package/dist/storage-backends/cursor-hosted.js +0 -30
- package/dist/storage-backends/file-kv.d.ts +9 -12
- package/dist/storage-backends/file-kv.d.ts.map +1 -1
- package/dist/storage-backends/file-kv.js +11 -47
- package/dist/storage-protocol.d.ts +3 -11
- package/dist/storage-protocol.d.ts.map +1 -1
- package/dist/storage-protocol.js +3 -11
- package/dist/storage.d.ts +8 -36
- package/dist/storage.d.ts.map +1 -1
- package/dist/storage.js +8 -44
- package/dist/types.d.ts +20 -57
- package/dist/types.d.ts.map +1 -1
- package/docs/README.md +10 -6
- package/docs/building-with-agents.md +3 -2
- package/docs/concepts.md +2 -4
- package/docs/evals.md +613 -298
- package/docs/guides/bitbucket.md +89 -0
- package/docs/guides/github.md +31 -0
- package/docs/guides/gitlab.md +97 -0
- package/docs/guides/webhooks.md +4 -2
- package/docs/hillclimbing.md +2 -2
- package/docs/reference/channels.md +16 -3
- package/docs/reference/cli.md +6 -4
- package/docs/reference/extensions.md +0 -3
- package/docs/reference/hooks.md +9 -12
- package/docs/reference/http-api.md +2 -3
- package/docs/reference/playground.md +0 -4
- package/docs/reference/project-layout.md +1 -8
- package/docs/reference/sessions.md +1 -2
- package/docs/skills/index.md +2 -2
- package/docs/storage.md +15 -24
- package/package.json +1 -7
- package/skills/deploy/SKILL.md +169 -0
- package/skills/evals/SKILL.md +45 -8
- package/skills/framework-map/SKILL.md +1 -2
- package/src/channels/bitbucket/api.ts +341 -0
- package/src/channels/bitbucket/binding.ts +25 -0
- package/src/channels/bitbucket/index.ts +2 -0
- package/src/channels/gitlab/api.ts +123 -0
- package/src/channels/gitlab/binding.ts +12 -0
- package/src/channels/gitlab/index.ts +1 -0
- package/src/channels/slack/dispatch.ts +30 -0
- package/src/channels/slack/slack-channel.ts +69 -7
- package/src/extensions.ts +2 -6
- package/src/index.ts +0 -3
- package/src/internal/authored-alias-hooks.ts +32 -0
- package/src/internal/authored-loaders.ts +26 -2
- package/src/internal/cli-ax.ts +1 -2
- package/src/internal/continuation-channel.ts +2 -1
- package/src/internal/continuation-identity.ts +10 -2
- package/src/internal/discovery/agent.ts +1 -7
- package/src/internal/discovery/extension-overlay.ts +0 -18
- package/src/internal/discovery/extensions.ts +2 -26
- package/src/internal/discovery/info.ts +3 -13
- package/src/internal/discovery/modules.ts +0 -1
- package/src/internal/discovery/project.ts +0 -16
- package/src/internal/eval-runner.ts +0 -1
- package/src/internal/framework-file-storage.ts +4 -5
- package/src/internal/framework-storage-selection.ts +2 -2
- package/src/internal/init-scaffold.ts +1 -2
- package/src/internal/install-cursor-skills.ts +38 -5
- package/src/internal/run-client.ts +1 -1
- package/src/internal/sdk-runner.ts +1 -1
- package/src/internal/server.ts +0 -16
- package/src/internal/session-engine.ts +14 -202
- package/src/internal/storage-coordinator.ts +5 -76
- package/src/internal/storage-roles.ts +4 -9
- package/src/storage-backends/cursor-hosted-v2.ts +4 -7
- package/src/storage-backends/cursor-hosted.ts +0 -37
- package/src/storage-backends/file-kv.ts +10 -51
- package/src/storage-protocol.ts +3 -17
- package/src/storage.ts +10 -101
- package/src/types.ts +19 -56
- package/templates/demo/README.md +10 -6
- package/templates/demo/agent/channels/github.ts +2 -0
- package/templates/demo/agent/channels/queue.ts +6 -2
- package/templates/demo/agent/lib/repos.ts +5 -0
- package/templates/demo/init.json +25 -0
- package/dist/ab.d.ts +0 -209
- package/dist/ab.d.ts.map +0 -1
- package/dist/ab.js +0 -246
- package/dist/docs/ab.html +0 -80
- package/dist/docs/ab.md +0 -332
- package/dist/docs/assets/ab.md.mlVgqvSk.js +0 -54
- package/dist/docs/assets/ab.md.mlVgqvSk.lean.js +0 -1
- package/dist/docs/assets/chunks/@localSearchIndexroot.BHZYsNVi.js +0 -1
- package/dist/docs/assets/concepts.md.DgEcZOfT.js +0 -1
- package/dist/docs/assets/evals.md.C1ekS3k2.js +0 -85
- package/dist/docs/assets/evals.md.C1ekS3k2.lean.js +0 -1
- package/dist/docs/assets/index.md.BjH1w2ZW.js +0 -5
- package/dist/docs/assets/reference_sessions.md.CueyOHSL.js +0 -1
- package/dist/docs/assets/skills_ab.md.CsFNatVx.js +0 -26
- package/dist/docs/assets/skills_ab.md.CsFNatVx.lean.js +0 -1
- package/dist/docs/assets/skills_evals.md.BhovOvrl.lean.js +0 -1
- package/dist/docs/assets/skills_index.md.DKwIxzGg.js +0 -1
- package/dist/docs/assets/skills_index.md.DKwIxzGg.lean.js +0 -1
- package/dist/docs/skills/ab.html +0 -52
- package/dist/docs/skills/ab.md +0 -50
- package/dist/internal/ab-collector.d.ts +0 -44
- package/dist/internal/ab-collector.d.ts.map +0 -1
- package/dist/internal/ab-collector.js +0 -142
- package/dist/internal/ab-fold.d.ts +0 -36
- package/dist/internal/ab-fold.d.ts.map +0 -1
- package/dist/internal/ab-fold.js +0 -175
- package/dist/internal/ab-snapshot.d.ts +0 -68
- package/dist/internal/ab-snapshot.d.ts.map +0 -1
- package/dist/internal/ab-snapshot.js +0 -208
- package/dist/internal/discovery/ab.d.ts +0 -9
- package/dist/internal/discovery/ab.d.ts.map +0 -1
- package/dist/internal/discovery/ab.js +0 -113
- package/dist/playground/assets/index-BMDqAeXu.js +0 -67
- package/dist/playground/assets/index-LUgJoWdL.css +0 -1
- package/docs/ab.md +0 -337
- package/skills/ab/SKILL.md +0 -58
- package/src/ab.ts +0 -430
- package/src/internal/ab-collector.ts +0 -200
- package/src/internal/ab-fold.ts +0 -232
- package/src/internal/ab-snapshot.ts +0 -331
- package/src/internal/discovery/ab.ts +0 -131
- /package/dist/docs/assets/{concepts.md.DgEcZOfT.lean.js → concepts.md.B4o63Gul.lean.js} +0 -0
- /package/dist/docs/assets/{guides_webhooks.md.DKdA43Qm.lean.js → guides_webhooks.md.CJK484ex.lean.js} +0 -0
- /package/dist/docs/assets/{hillclimbing.md.CpTGTCle.lean.js → hillclimbing.md.BOiVo1tf.lean.js} +0 -0
- /package/dist/docs/assets/{reference_cli.md.Ca26u0Es.lean.js → reference_cli.md.Deg7849l.lean.js} +0 -0
- /package/dist/docs/assets/{reference_extensions.md.CPWt00ds.lean.js → reference_extensions.md.ZAVUyuEX.lean.js} +0 -0
- /package/dist/docs/assets/{reference_http-api.md.oySXBO8o.lean.js → reference_http-api.md.BwaCo-VO.lean.js} +0 -0
- /package/dist/docs/assets/{reference_sessions.md.CueyOHSL.lean.js → reference_sessions.md.CyXV1MUw.lean.js} +0 -0
- /package/dist/docs/assets/{skills_framework-map.md.D-tFZhFS.lean.js → skills_framework-map.md.BxLSOhSY.lean.js} +0 -0
package/dist/ab.d.ts
DELETED
|
@@ -1,209 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Live A/B metrics plug-in for agent-serve.
|
|
3
|
-
*
|
|
4
|
-
* Author `agent/ab.ts` (or `agent/ab/<name>.ts`) with {@link defineAB}. At
|
|
5
|
-
* session creation the engine runs {@link ABConfig.split} and appends durable
|
|
6
|
-
* `ab.assigned` events; arms are exposed on {@link SessionInfo.abs} for the
|
|
7
|
-
* agent loop (tools / optional per-variant `instructions`). {@link ABCollector}
|
|
8
|
-
* folds the event stream into metrics and calls {@link ABConfig.onSample}.
|
|
9
|
-
* There is no `agent-sdk ab` test runner.
|
|
10
|
-
*
|
|
11
|
-
* Assignments and foldable metrics live in each session's `events.ndjson`
|
|
12
|
-
* (survive serve restart). Optional project defaults in `agent/ab.config.ts`
|
|
13
|
-
* ({@link defineABConfig}) cap the playground fold window; durable
|
|
14
|
-
* sample/snapshot export lives on the `defineStorage` `abs` table.
|
|
15
|
-
*
|
|
16
|
-
* ```ts
|
|
17
|
-
* import { defineAB, splitBySessionHash } from "@cursor/july/ab";
|
|
18
|
-
*
|
|
19
|
-
* export default defineAB({
|
|
20
|
-
* name: "concise-instructions",
|
|
21
|
-
* variants: {
|
|
22
|
-
* control: { label: "Baseline" },
|
|
23
|
-
* treatment: { label: "Shorter" },
|
|
24
|
-
* },
|
|
25
|
-
* split: splitBySessionHash({ holdout: 0.1 }),
|
|
26
|
-
* onSample(sample) {
|
|
27
|
-
* console.log(sample.variant, sample.metrics);
|
|
28
|
-
* },
|
|
29
|
-
* });
|
|
30
|
-
* ```
|
|
31
|
-
*/
|
|
32
|
-
import type { SessionEvent, SessionInfo } from "./types.js";
|
|
33
|
-
export type { ABSnapshot } from "./internal/ab-snapshot.js";
|
|
34
|
-
export interface ABVariantMeta {
|
|
35
|
-
label?: string;
|
|
36
|
-
description?: string;
|
|
37
|
-
/**
|
|
38
|
-
* Optional prompt overlay for this arm. Appended to the agent's
|
|
39
|
-
* instructions for the session (local `AGENTS.md` + cloud first-turn
|
|
40
|
-
* preamble) after sticky enrollment.
|
|
41
|
-
*/
|
|
42
|
-
instructions?: string;
|
|
43
|
-
}
|
|
44
|
-
export interface ABSplitContext {
|
|
45
|
-
agent: {
|
|
46
|
-
name: string;
|
|
47
|
-
};
|
|
48
|
-
channel: {
|
|
49
|
-
id: string;
|
|
50
|
-
};
|
|
51
|
-
/**
|
|
52
|
-
* Live sessions only — the engine never calls `split` for
|
|
53
|
-
* `purpose: "eval"` harness sessions.
|
|
54
|
-
*/
|
|
55
|
-
session: SessionInfo;
|
|
56
|
-
experiment: string;
|
|
57
|
-
/** Variant keys from {@link ABConfig.variants}, in declaration order. */
|
|
58
|
-
variants: string[];
|
|
59
|
-
}
|
|
60
|
-
export interface ABCollectContext {
|
|
61
|
-
agent: {
|
|
62
|
-
name: string;
|
|
63
|
-
};
|
|
64
|
-
channel: {
|
|
65
|
-
id: string;
|
|
66
|
-
};
|
|
67
|
-
session: SessionInfo;
|
|
68
|
-
experiment: string;
|
|
69
|
-
variant: string;
|
|
70
|
-
}
|
|
71
|
-
/** Cumulative per-session metrics for one experiment arm. */
|
|
72
|
-
export interface ABMetrics {
|
|
73
|
-
turns: number;
|
|
74
|
-
turnFailures: number;
|
|
75
|
-
toolCalls: number;
|
|
76
|
-
toolErrors: number;
|
|
77
|
-
inputTokens: number;
|
|
78
|
-
outputTokens: number;
|
|
79
|
-
cacheReadTokens: number;
|
|
80
|
-
cacheWriteTokens: number;
|
|
81
|
-
/**
|
|
82
|
-
* Sum of estimated turn costs in USD (the `cost.totalUsd` recorded on
|
|
83
|
-
* `turn.completed` events). Turns whose model has no known rates
|
|
84
|
-
* contribute 0.
|
|
85
|
-
*/
|
|
86
|
-
costUsd: number;
|
|
87
|
-
/** Sum of turn wall times (turn.started → turn.completed/failed). */
|
|
88
|
-
wallTimeMs: number;
|
|
89
|
-
/** Custom counters from {@link ABConfig.derive}. */
|
|
90
|
-
custom: Record<string, number>;
|
|
91
|
-
}
|
|
92
|
-
export type ABSampleReason = "turn.completed" | "turn.failed";
|
|
93
|
-
export interface ABMetricSample {
|
|
94
|
-
experiment: string;
|
|
95
|
-
variant: string;
|
|
96
|
-
variantLabel?: string;
|
|
97
|
-
sessionId: string;
|
|
98
|
-
channelId: string;
|
|
99
|
-
metrics: ABMetrics;
|
|
100
|
-
reason: ABSampleReason;
|
|
101
|
-
at: string;
|
|
102
|
-
}
|
|
103
|
-
/** Variant key to enroll, or `null` to leave the session out (sticky). */
|
|
104
|
-
export type ABSplitResult = string | null;
|
|
105
|
-
/**
|
|
106
|
-
* Sticky enrollment decision. Return a variant key, or `null` to skip.
|
|
107
|
-
* Prefer helpers ({@link splitBySessionHash}, {@link splitIf}, …).
|
|
108
|
-
*/
|
|
109
|
-
export type ABSplit = (ctx: ABSplitContext) => ABSplitResult | Promise<ABSplitResult>;
|
|
110
|
-
export interface ABSplitOptions {
|
|
111
|
-
/** Relative weights per variant key (default: equal). */
|
|
112
|
-
weights?: Record<string, number>;
|
|
113
|
-
/** Fraction of sessions left out (`0`–`1`, sticky). */
|
|
114
|
-
holdout?: number;
|
|
115
|
-
/** Salt so a new rollout reshuffles without renaming the experiment. */
|
|
116
|
-
salt?: string;
|
|
117
|
-
}
|
|
118
|
-
/** Sticky hash of session id → arm (recommended default). */
|
|
119
|
-
export declare function splitBySessionHash(options?: ABSplitOptions): ABSplit;
|
|
120
|
-
/** Random draw once per new session (then sticky via persistence). */
|
|
121
|
-
export declare function splitByRandom(options?: Omit<ABSplitOptions, "salt">): ABSplit;
|
|
122
|
-
/** Pin every new session to one arm. */
|
|
123
|
-
export declare function splitAlways(variant: string): ABSplit;
|
|
124
|
-
/** Skip every session (pause without deleting the experiment). */
|
|
125
|
-
export declare function splitNone(): ABSplit;
|
|
126
|
-
/** Run `inner` only when `predicate` is true; otherwise skip. */
|
|
127
|
-
export declare function splitIf(predicate: (ctx: ABSplitContext) => boolean | Promise<boolean>, inner: ABSplit): ABSplit;
|
|
128
|
-
export interface ABConfig {
|
|
129
|
-
/** Experiment id. Defaults to the file stem (`ab/concise.ts` → `concise`). */
|
|
130
|
-
name?: string;
|
|
131
|
-
/**
|
|
132
|
-
* Named arms (at least two). Prefer `control` / `treatment` over `a` / `b`.
|
|
133
|
-
*/
|
|
134
|
-
variants: Record<string, ABVariantMeta>;
|
|
135
|
-
/**
|
|
136
|
-
* Choose an arm at session creation, or return `null` to skip. The engine
|
|
137
|
-
* appends a durable `ab.assigned` event; the collector folds that stream.
|
|
138
|
-
*/
|
|
139
|
-
split: ABSplit;
|
|
140
|
-
/**
|
|
141
|
-
* Optional custom metric extractors. Return a numeric delta, or
|
|
142
|
-
* `null`/`undefined` to skip. Called on **every** session event (including
|
|
143
|
-
* high-frequency `message.appended` deltas) — keep these synchronous and
|
|
144
|
-
* cheap; do not await I/O.
|
|
145
|
-
*/
|
|
146
|
-
derive?: Record<string, (event: SessionEvent, ctx: ABCollectContext) => number | null | undefined | Promise<number | null | undefined>>;
|
|
147
|
-
/**
|
|
148
|
-
* Called on turn complete/fail for enrolled sessions only. Observe-only;
|
|
149
|
-
* errors are logged and never fatal.
|
|
150
|
-
*/
|
|
151
|
-
onSample: (sample: ABMetricSample) => void | Promise<void>;
|
|
152
|
-
}
|
|
153
|
-
export type ABDefinition = ABConfig & {
|
|
154
|
-
readonly __agentServe: "ab";
|
|
155
|
-
};
|
|
156
|
-
/** Author a live A/B metrics experiment under `agent/ab`. */
|
|
157
|
-
export declare function defineAB(config: ABConfig): ABDefinition;
|
|
158
|
-
/** Resolve {@link ABConfig.split} to a variant key or `null`. */
|
|
159
|
-
export declare function resolveABSplit(split: ABSplit, ctx: ABSplitContext): Promise<ABSplitResult>;
|
|
160
|
-
/** FNV-1a → [0, 1). */
|
|
161
|
-
export declare function hashToUnitInterval(input: string): number;
|
|
162
|
-
export declare function pickWeightedVariant(variantKeys: readonly string[], weights: Record<string, number> | undefined, unitInterval: number): string;
|
|
163
|
-
export declare function emptyABMetrics(): ABMetrics;
|
|
164
|
-
export declare function cloneABMetrics(metrics: ABMetrics): ABMetrics;
|
|
165
|
-
/**
|
|
166
|
-
* Append enrolled per-variant `instructions` overlays onto the base agent
|
|
167
|
-
* prompt. Used by the session engine before materializing the harness.
|
|
168
|
-
*/
|
|
169
|
-
export declare function mergeABInstructions(base: string | undefined, abs: Readonly<Record<string, string | null>> | undefined, experiments: ReadonlyArray<{
|
|
170
|
-
name: string;
|
|
171
|
-
definition: ABDefinition;
|
|
172
|
-
}>): string | undefined;
|
|
173
|
-
/** Default sessions folded into `GET /v1/abs` / the A/Bs tab. */
|
|
174
|
-
export declare const AB_DEFAULT_MAX_PLAYGROUND_SESSIONS = 200;
|
|
175
|
-
/** Hard ceiling for {@link ABConfigFile.maxPlaygroundSessions}. */
|
|
176
|
-
export declare const AB_MAX_PLAYGROUND_SESSIONS = 10000;
|
|
177
|
-
/**
|
|
178
|
-
* Project-wide A/B defaults (`agent/ab.config.ts`).
|
|
179
|
-
*
|
|
180
|
-
* **Assignments and foldable metrics** already persist in each session's
|
|
181
|
-
* `events.ndjson` under `--state-root`; durable sample/snapshot export
|
|
182
|
-
* lives on the `defineStorage` `abs` table (`agent/storage.ts`). This
|
|
183
|
-
* config only caps how many sessions the playground / `GET /v1/abs` folds.
|
|
184
|
-
*/
|
|
185
|
-
export interface ABConfigFile {
|
|
186
|
-
/**
|
|
187
|
-
* Optional. Max newest sessions folded into **`GET /v1/abs` / the A/Bs
|
|
188
|
-
* tab**. Does not prune session logs or change assignment.
|
|
189
|
-
*
|
|
190
|
-
* Default {@link AB_DEFAULT_MAX_PLAYGROUND_SESSIONS}; hard-capped at
|
|
191
|
-
* {@link AB_MAX_PLAYGROUND_SESSIONS}.
|
|
192
|
-
*/
|
|
193
|
-
maxPlaygroundSessions?: number;
|
|
194
|
-
}
|
|
195
|
-
/**
|
|
196
|
-
* Defaults for every experiment under `agent/ab*` (`agent/ab.config.ts`).
|
|
197
|
-
*
|
|
198
|
-
* Session event logs remain the assignment/metrics source of truth;
|
|
199
|
-
* durable exports go through the `defineStorage` `abs` table.
|
|
200
|
-
*/
|
|
201
|
-
export declare function defineABConfig(config: ABConfigFile): ABConfigFile;
|
|
202
|
-
/** Validate {@link ABConfigFile.maxPlaygroundSessions}. */
|
|
203
|
-
export declare function normalizeABMaxPlaygroundSessions(value: unknown): number;
|
|
204
|
-
/**
|
|
205
|
-
* Effective `GET /v1/abs` fold window. Invalid values fall back to
|
|
206
|
-
* {@link AB_DEFAULT_MAX_PLAYGROUND_SESSIONS}.
|
|
207
|
-
*/
|
|
208
|
-
export declare function resolveABMaxPlaygroundSessions(value: unknown): number;
|
|
209
|
-
//# sourceMappingURL=ab.d.ts.map
|
package/dist/ab.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ab.d.ts","sourceRoot":"","sources":["../src/ab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE5D,YAAY,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAE5D,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB;;;OAGG;IACH,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,6DAA6D;AAC7D,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,qEAAqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAED,MAAM,MAAM,cAAc,GAAG,gBAAgB,GAAG,aAAa,CAAC;AAE9D,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,SAAS,CAAC;IACnB,MAAM,EAAE,cAAc,CAAC;IACvB,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,0EAA0E;AAC1E,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,IAAI,CAAC;AAE1C;;;GAGG;AACH,MAAM,MAAM,OAAO,GAAG,CACpB,GAAG,EAAE,cAAc,KAChB,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;AAE5C,MAAM,WAAW,cAAc;IAC7B,yDAAyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,6DAA6D;AAC7D,wBAAgB,kBAAkB,CAAC,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CASpE;AAED,sEAAsE;AACtE,wBAAgB,aAAa,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,GAAG,OAAO,CAI7E;AAED,wCAAwC;AACxC,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAEpD;AAED,kEAAkE;AAClE,wBAAgB,SAAS,IAAI,OAAO,CAEnC;AAED,iEAAiE;AACjE,wBAAgB,OAAO,CACrB,SAAS,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,EAC9D,KAAK,EAAE,OAAO,GACb,OAAO,CAET;AAED,MAAM,WAAW,QAAQ;IACvB,8EAA8E;IAC9E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACxC;;;OAGG;IACH,KAAK,EAAE,OAAO,CAAC;IACf;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CACb,MAAM,EACN,CACE,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,gBAAgB,KAClB,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,CACpE,CAAC;IACF;;;OAGG;IACH,QAAQ,EAAE,CAAC,MAAM,EAAE,cAAc,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5D;AAED,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG;IAAE,QAAQ,CAAC,YAAY,EAAE,IAAI,CAAA;CAAE,CAAC;AAEtE,6DAA6D;AAC7D,wBAAgB,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,YAAY,CA8BvD;AAED,iEAAiE;AACjE,wBAAsB,cAAc,CAClC,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,cAAc,GAClB,OAAO,CAAC,aAAa,CAAC,CAWxB;AA2BD,uBAAuB;AACvB,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAOxD;AAED,wBAAgB,mBAAmB,CACjC,WAAW,EAAE,SAAS,MAAM,EAAE,EAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,EAC3C,YAAY,EAAE,MAAM,GACnB,MAAM,CAsBR;AAMD,wBAAgB,cAAc,IAAI,SAAS,CAc1C;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,SAAS,GAAG,SAAS,CAE5D;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,GAAG,SAAS,EACxB,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,GAAG,SAAS,EACxD,WAAW,EAAE,aAAa,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,YAAY,CAAA;CAAE,CAAC,GACrE,MAAM,GAAG,SAAS,CAsBpB;AAMD,iEAAiE;AACjE,eAAO,MAAM,kCAAkC,MAAM,CAAC;AAEtD,mEAAmE;AACnE,eAAO,MAAM,0BAA0B,QAAS,CAAC;AAEjD;;;;;;;GAOG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,YAAY,GAAG,YAAY,CAMjE;AAED,2DAA2D;AAC3D,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAKvE;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASrE"}
|
package/dist/ab.js
DELETED
|
@@ -1,246 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Live A/B metrics plug-in for agent-serve.
|
|
3
|
-
*
|
|
4
|
-
* Author `agent/ab.ts` (or `agent/ab/<name>.ts`) with {@link defineAB}. At
|
|
5
|
-
* session creation the engine runs {@link ABConfig.split} and appends durable
|
|
6
|
-
* `ab.assigned` events; arms are exposed on {@link SessionInfo.abs} for the
|
|
7
|
-
* agent loop (tools / optional per-variant `instructions`). {@link ABCollector}
|
|
8
|
-
* folds the event stream into metrics and calls {@link ABConfig.onSample}.
|
|
9
|
-
* There is no `agent-sdk ab` test runner.
|
|
10
|
-
*
|
|
11
|
-
* Assignments and foldable metrics live in each session's `events.ndjson`
|
|
12
|
-
* (survive serve restart). Optional project defaults in `agent/ab.config.ts`
|
|
13
|
-
* ({@link defineABConfig}) cap the playground fold window; durable
|
|
14
|
-
* sample/snapshot export lives on the `defineStorage` `abs` table.
|
|
15
|
-
*
|
|
16
|
-
* ```ts
|
|
17
|
-
* import { defineAB, splitBySessionHash } from "@cursor/july/ab";
|
|
18
|
-
*
|
|
19
|
-
* export default defineAB({
|
|
20
|
-
* name: "concise-instructions",
|
|
21
|
-
* variants: {
|
|
22
|
-
* control: { label: "Baseline" },
|
|
23
|
-
* treatment: { label: "Shorter" },
|
|
24
|
-
* },
|
|
25
|
-
* split: splitBySessionHash({ holdout: 0.1 }),
|
|
26
|
-
* onSample(sample) {
|
|
27
|
-
* console.log(sample.variant, sample.metrics);
|
|
28
|
-
* },
|
|
29
|
-
* });
|
|
30
|
-
* ```
|
|
31
|
-
*/
|
|
32
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
33
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
34
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
35
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
36
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
37
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
38
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
39
|
-
});
|
|
40
|
-
};
|
|
41
|
-
import { normalizePositiveInt } from "./internal/bounded-int.js";
|
|
42
|
-
import { brandDefinition } from "./internal/brand.js";
|
|
43
|
-
/** Sticky hash of session id → arm (recommended default). */
|
|
44
|
-
export function splitBySessionHash(options) {
|
|
45
|
-
assertHoldout(options === null || options === void 0 ? void 0 : options.holdout);
|
|
46
|
-
return (ctx) => { var _a; return pickArm(ctx.variants, options === null || options === void 0 ? void 0 : options.weights, hashToUnitInterval(`${(_a = options === null || options === void 0 ? void 0 : options.salt) !== null && _a !== void 0 ? _a : ""}\0${ctx.session.id}`), options === null || options === void 0 ? void 0 : options.holdout); };
|
|
47
|
-
}
|
|
48
|
-
/** Random draw once per new session (then sticky via persistence). */
|
|
49
|
-
export function splitByRandom(options) {
|
|
50
|
-
assertHoldout(options === null || options === void 0 ? void 0 : options.holdout);
|
|
51
|
-
return (ctx) => pickArm(ctx.variants, options === null || options === void 0 ? void 0 : options.weights, Math.random(), options === null || options === void 0 ? void 0 : options.holdout);
|
|
52
|
-
}
|
|
53
|
-
/** Pin every new session to one arm. */
|
|
54
|
-
export function splitAlways(variant) {
|
|
55
|
-
return () => variant;
|
|
56
|
-
}
|
|
57
|
-
/** Skip every session (pause without deleting the experiment). */
|
|
58
|
-
export function splitNone() {
|
|
59
|
-
return () => null;
|
|
60
|
-
}
|
|
61
|
-
/** Run `inner` only when `predicate` is true; otherwise skip. */
|
|
62
|
-
export function splitIf(predicate, inner) {
|
|
63
|
-
return (ctx) => __awaiter(this, void 0, void 0, function* () { return ((yield predicate(ctx)) ? inner(ctx) : null); });
|
|
64
|
-
}
|
|
65
|
-
/** Author a live A/B metrics experiment under `agent/ab`. */
|
|
66
|
-
export function defineAB(config) {
|
|
67
|
-
var _a;
|
|
68
|
-
const keys = Object.keys((_a = config.variants) !== null && _a !== void 0 ? _a : {});
|
|
69
|
-
if (keys.length < 2) {
|
|
70
|
-
throw new Error(`defineAB: variants needs at least two arms (got ${keys.length})`);
|
|
71
|
-
}
|
|
72
|
-
for (const key of keys) {
|
|
73
|
-
if (key.length === 0 || key.includes("/") || key.includes("\\")) {
|
|
74
|
-
throw new Error(`defineAB: invalid variant key ${JSON.stringify(key)} (single segment)`);
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
if (typeof config.split !== "function") {
|
|
78
|
-
throw new Error("defineAB: split must be a function (splitBySessionHash(), splitIf(...), or custom)");
|
|
79
|
-
}
|
|
80
|
-
if (typeof config.onSample !== "function") {
|
|
81
|
-
throw new Error("defineAB: onSample(sample) is required");
|
|
82
|
-
}
|
|
83
|
-
if (config.derive !== undefined) {
|
|
84
|
-
for (const [key, fn] of Object.entries(config.derive)) {
|
|
85
|
-
if (typeof fn !== "function") {
|
|
86
|
-
throw new Error(`defineAB: derive.${key} must be a function`);
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
return brandDefinition("ab", config);
|
|
91
|
-
}
|
|
92
|
-
/** Resolve {@link ABConfig.split} to a variant key or `null`. */
|
|
93
|
-
export function resolveABSplit(split, ctx) {
|
|
94
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
95
|
-
const result = yield split(ctx);
|
|
96
|
-
if (result === null || result === undefined) {
|
|
97
|
-
return null;
|
|
98
|
-
}
|
|
99
|
-
if (typeof result === "string") {
|
|
100
|
-
return result;
|
|
101
|
-
}
|
|
102
|
-
throw new Error(`defineAB: split must return a variant key or null (got ${typeof result})`);
|
|
103
|
-
});
|
|
104
|
-
}
|
|
105
|
-
function assertHoldout(holdout) {
|
|
106
|
-
if (holdout === undefined) {
|
|
107
|
-
return;
|
|
108
|
-
}
|
|
109
|
-
if (!Number.isFinite(holdout) || holdout < 0 || holdout > 1) {
|
|
110
|
-
throw new Error("defineAB: holdout must be a finite number in [0, 1]");
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
/** Pick a variant (or `null` via holdout) from a unit interval in `[0, 1)`. */
|
|
114
|
-
function pickArm(variantKeys, weights, unitInterval, holdout) {
|
|
115
|
-
const u = clampUnit(unitInterval);
|
|
116
|
-
const out = holdout !== null && holdout !== void 0 ? holdout : 0;
|
|
117
|
-
if (out >= 1 || (out > 0 && u < out)) {
|
|
118
|
-
return null;
|
|
119
|
-
}
|
|
120
|
-
const remapped = out > 0 ? (u - out) / (1 - out) : u;
|
|
121
|
-
return pickWeightedVariant(variantKeys, weights, remapped);
|
|
122
|
-
}
|
|
123
|
-
/** FNV-1a → [0, 1). */
|
|
124
|
-
export function hashToUnitInterval(input) {
|
|
125
|
-
let h = 2166136261;
|
|
126
|
-
for (let i = 0; i < input.length; i++) {
|
|
127
|
-
h ^= input.charCodeAt(i);
|
|
128
|
-
h = Math.imul(h, 16777619);
|
|
129
|
-
}
|
|
130
|
-
return (h >>> 0) / 4294967296;
|
|
131
|
-
}
|
|
132
|
-
export function pickWeightedVariant(variantKeys, weights, unitInterval) {
|
|
133
|
-
var _a;
|
|
134
|
-
const u = clampUnit(unitInterval);
|
|
135
|
-
const rows = [];
|
|
136
|
-
let total = 0;
|
|
137
|
-
for (const key of variantKeys) {
|
|
138
|
-
const weight = (_a = weights === null || weights === void 0 ? void 0 : weights[key]) !== null && _a !== void 0 ? _a : 1;
|
|
139
|
-
if (weight > 0 && Number.isFinite(weight)) {
|
|
140
|
-
rows.push({ key, weight });
|
|
141
|
-
total += weight;
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
if (rows.length === 0 || total <= 0) {
|
|
145
|
-
throw new Error("defineAB: no positive-weight variants to split across");
|
|
146
|
-
}
|
|
147
|
-
let cursor = u * total;
|
|
148
|
-
for (const row of rows) {
|
|
149
|
-
cursor -= row.weight;
|
|
150
|
-
if (cursor < 0) {
|
|
151
|
-
return row.key;
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
return rows[rows.length - 1].key;
|
|
155
|
-
}
|
|
156
|
-
function clampUnit(unitInterval) {
|
|
157
|
-
return Math.min(Math.max(unitInterval, 0), 0.999999999);
|
|
158
|
-
}
|
|
159
|
-
export function emptyABMetrics() {
|
|
160
|
-
return {
|
|
161
|
-
turns: 0,
|
|
162
|
-
turnFailures: 0,
|
|
163
|
-
toolCalls: 0,
|
|
164
|
-
toolErrors: 0,
|
|
165
|
-
inputTokens: 0,
|
|
166
|
-
outputTokens: 0,
|
|
167
|
-
cacheReadTokens: 0,
|
|
168
|
-
cacheWriteTokens: 0,
|
|
169
|
-
costUsd: 0,
|
|
170
|
-
wallTimeMs: 0,
|
|
171
|
-
custom: {},
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
|
-
export function cloneABMetrics(metrics) {
|
|
175
|
-
return Object.assign(Object.assign({}, metrics), { custom: Object.assign({}, metrics.custom) });
|
|
176
|
-
}
|
|
177
|
-
/**
|
|
178
|
-
* Append enrolled per-variant `instructions` overlays onto the base agent
|
|
179
|
-
* prompt. Used by the session engine before materializing the harness.
|
|
180
|
-
*/
|
|
181
|
-
export function mergeABInstructions(base, abs, experiments) {
|
|
182
|
-
var _a;
|
|
183
|
-
var _b;
|
|
184
|
-
if (abs === undefined) {
|
|
185
|
-
return base;
|
|
186
|
-
}
|
|
187
|
-
const extras = [];
|
|
188
|
-
for (const experiment of experiments) {
|
|
189
|
-
const variant = abs[experiment.name];
|
|
190
|
-
if (variant === undefined || variant === null) {
|
|
191
|
-
continue;
|
|
192
|
-
}
|
|
193
|
-
const extra = (_a = experiment.definition.variants[variant]) === null || _a === void 0 ? void 0 : _a.instructions;
|
|
194
|
-
if (typeof extra === "string" && extra.trim() !== "") {
|
|
195
|
-
extras.push(extra.trim());
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
if (extras.length === 0) {
|
|
199
|
-
return base;
|
|
200
|
-
}
|
|
201
|
-
const baseText = (_b = base === null || base === void 0 ? void 0 : base.trim()) !== null && _b !== void 0 ? _b : "";
|
|
202
|
-
return baseText === ""
|
|
203
|
-
? extras.join("\n\n")
|
|
204
|
-
: `${baseText}\n\n${extras.join("\n\n")}`;
|
|
205
|
-
}
|
|
206
|
-
// ---------------------------------------------------------------------------
|
|
207
|
-
// Project config (`agent/ab.config.ts`) — playground fold window
|
|
208
|
-
// ---------------------------------------------------------------------------
|
|
209
|
-
/** Default sessions folded into `GET /v1/abs` / the A/Bs tab. */
|
|
210
|
-
export const AB_DEFAULT_MAX_PLAYGROUND_SESSIONS = 200;
|
|
211
|
-
/** Hard ceiling for {@link ABConfigFile.maxPlaygroundSessions}. */
|
|
212
|
-
export const AB_MAX_PLAYGROUND_SESSIONS = 10000;
|
|
213
|
-
/**
|
|
214
|
-
* Defaults for every experiment under `agent/ab*` (`agent/ab.config.ts`).
|
|
215
|
-
*
|
|
216
|
-
* Session event logs remain the assignment/metrics source of truth;
|
|
217
|
-
* durable exports go through the `defineStorage` `abs` table.
|
|
218
|
-
*/
|
|
219
|
-
export function defineABConfig(config) {
|
|
220
|
-
var _a;
|
|
221
|
-
return {
|
|
222
|
-
maxPlaygroundSessions: normalizeABMaxPlaygroundSessions((_a = config.maxPlaygroundSessions) !== null && _a !== void 0 ? _a : AB_DEFAULT_MAX_PLAYGROUND_SESSIONS),
|
|
223
|
-
};
|
|
224
|
-
}
|
|
225
|
-
/** Validate {@link ABConfigFile.maxPlaygroundSessions}. */
|
|
226
|
-
export function normalizeABMaxPlaygroundSessions(value) {
|
|
227
|
-
return normalizePositiveInt(value, {
|
|
228
|
-
label: "ab.config.ts: maxPlaygroundSessions",
|
|
229
|
-
max: AB_MAX_PLAYGROUND_SESSIONS,
|
|
230
|
-
});
|
|
231
|
-
}
|
|
232
|
-
/**
|
|
233
|
-
* Effective `GET /v1/abs` fold window. Invalid values fall back to
|
|
234
|
-
* {@link AB_DEFAULT_MAX_PLAYGROUND_SESSIONS}.
|
|
235
|
-
*/
|
|
236
|
-
export function resolveABMaxPlaygroundSessions(value) {
|
|
237
|
-
if (value === undefined) {
|
|
238
|
-
return AB_DEFAULT_MAX_PLAYGROUND_SESSIONS;
|
|
239
|
-
}
|
|
240
|
-
try {
|
|
241
|
-
return normalizeABMaxPlaygroundSessions(value);
|
|
242
|
-
}
|
|
243
|
-
catch (_a) {
|
|
244
|
-
return AB_DEFAULT_MAX_PLAYGROUND_SESSIONS;
|
|
245
|
-
}
|
|
246
|
-
}
|