@stigmer/cli 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/auth/browser.d.ts +7 -0
  2. package/auth/browser.d.ts.map +1 -0
  3. package/auth/browser.js +49 -0
  4. package/auth/browser.js.map +1 -0
  5. package/auth/callback-server.d.ts +8 -3
  6. package/auth/callback-server.d.ts.map +1 -1
  7. package/auth/callback-server.js +53 -19
  8. package/auth/callback-server.js.map +1 -1
  9. package/auth/device.d.ts +23 -0
  10. package/auth/device.d.ts.map +1 -0
  11. package/auth/device.js +119 -0
  12. package/auth/device.js.map +1 -0
  13. package/auth/index.d.ts +4 -3
  14. package/auth/index.d.ts.map +1 -1
  15. package/auth/index.js +4 -3
  16. package/auth/index.js.map +1 -1
  17. package/auth/login.d.ts +49 -1
  18. package/auth/login.d.ts.map +1 -1
  19. package/auth/login.js +161 -51
  20. package/auth/login.js.map +1 -1
  21. package/auth/oidc.d.ts +67 -0
  22. package/auth/oidc.d.ts.map +1 -0
  23. package/auth/oidc.js +213 -0
  24. package/auth/oidc.js.map +1 -0
  25. package/auth/token.d.ts +17 -23
  26. package/auth/token.d.ts.map +1 -1
  27. package/auth/token.js +51 -60
  28. package/auth/token.js.map +1 -1
  29. package/client/client.d.ts.map +1 -1
  30. package/client/client.js +4 -1
  31. package/client/client.js.map +1 -1
  32. package/client/current-organization.d.ts +5 -0
  33. package/client/current-organization.d.ts.map +1 -0
  34. package/client/current-organization.js +15 -0
  35. package/client/current-organization.js.map +1 -0
  36. package/commands/apikey/index.d.ts +16 -1
  37. package/commands/apikey/index.d.ts.map +1 -1
  38. package/commands/apikey/index.js +81 -0
  39. package/commands/apikey/index.js.map +1 -1
  40. package/commands/app-handover.d.ts +17 -0
  41. package/commands/app-handover.d.ts.map +1 -0
  42. package/commands/app-handover.js +155 -0
  43. package/commands/app-handover.js.map +1 -0
  44. package/commands/auth/index.d.ts.map +1 -1
  45. package/commands/auth/index.js +15 -9
  46. package/commands/auth/index.js.map +1 -1
  47. package/commands/auth/login.d.ts +7 -2
  48. package/commands/auth/login.d.ts.map +1 -1
  49. package/commands/auth/login.js +26 -23
  50. package/commands/auth/login.js.map +1 -1
  51. package/commands/auth/whoami.d.ts +5 -0
  52. package/commands/auth/whoami.d.ts.map +1 -1
  53. package/commands/auth/whoami.js +13 -1
  54. package/commands/auth/whoami.js.map +1 -1
  55. package/commands/coding-agent.d.ts +5 -0
  56. package/commands/coding-agent.d.ts.map +1 -0
  57. package/commands/coding-agent.js +30 -0
  58. package/commands/coding-agent.js.map +1 -0
  59. package/commands/config/backend.js +20 -6
  60. package/commands/config/backend.js.map +1 -1
  61. package/commands/embedding.d.ts +28 -0
  62. package/commands/embedding.d.ts.map +1 -0
  63. package/commands/embedding.js +170 -0
  64. package/commands/embedding.js.map +1 -0
  65. package/commands/example.d.ts +13 -0
  66. package/commands/example.d.ts.map +1 -0
  67. package/commands/example.js +126 -0
  68. package/commands/example.js.map +1 -0
  69. package/commands/help.d.ts.map +1 -1
  70. package/commands/help.js +16 -3
  71. package/commands/help.js.map +1 -1
  72. package/commands/load-test/burst.d.ts +64 -0
  73. package/commands/load-test/burst.d.ts.map +1 -0
  74. package/commands/load-test/burst.js +202 -0
  75. package/commands/load-test/burst.js.map +1 -0
  76. package/commands/load-test/index.d.ts +37 -0
  77. package/commands/load-test/index.d.ts.map +1 -0
  78. package/commands/load-test/index.js +154 -0
  79. package/commands/load-test/index.js.map +1 -0
  80. package/commands/load-test/report.d.ts +98 -0
  81. package/commands/load-test/report.d.ts.map +1 -0
  82. package/commands/load-test/report.js +210 -0
  83. package/commands/load-test/report.js.map +1 -0
  84. package/commands/mcp-server.d.ts +3 -3
  85. package/commands/mcp-server.d.ts.map +1 -1
  86. package/commands/mcp-server.js +4 -4
  87. package/commands/mcp-server.js.map +1 -1
  88. package/commands/setup.d.ts +17 -0
  89. package/commands/setup.d.ts.map +1 -1
  90. package/commands/setup.js +71 -1
  91. package/commands/setup.js.map +1 -1
  92. package/commands/up.d.ts.map +1 -1
  93. package/commands/up.js +67 -9
  94. package/commands/up.js.map +1 -1
  95. package/commands/usage.d.ts +15 -0
  96. package/commands/usage.d.ts.map +1 -1
  97. package/commands/usage.js +52 -2
  98. package/commands/usage.js.map +1 -1
  99. package/config/config.d.ts +14 -7
  100. package/config/config.d.ts.map +1 -1
  101. package/config/config.js +5 -1
  102. package/config/config.js.map +1 -1
  103. package/config/index.d.ts +1 -1
  104. package/config/index.d.ts.map +1 -1
  105. package/config/index.js +1 -1
  106. package/config/index.js.map +1 -1
  107. package/config/resolve.d.ts +1 -1
  108. package/config/resolve.d.ts.map +1 -1
  109. package/config/resolve.js +6 -3
  110. package/config/resolve.js.map +1 -1
  111. package/errors/classify.d.ts.map +1 -1
  112. package/errors/classify.js +10 -1
  113. package/errors/classify.js.map +1 -1
  114. package/errors/cli-exit-error.d.ts +3 -1
  115. package/errors/cli-exit-error.d.ts.map +1 -1
  116. package/errors/cli-exit-error.js +6 -2
  117. package/errors/cli-exit-error.js.map +1 -1
  118. package/errors/handle.d.ts +32 -5
  119. package/errors/handle.d.ts.map +1 -1
  120. package/errors/handle.js +67 -8
  121. package/errors/handle.js.map +1 -1
  122. package/local/daemon/components.d.ts +28 -1
  123. package/local/daemon/components.d.ts.map +1 -1
  124. package/local/daemon/components.js +64 -2
  125. package/local/daemon/components.js.map +1 -1
  126. package/local/daemon/host.d.ts.map +1 -1
  127. package/local/daemon/host.js +6 -2
  128. package/local/daemon/host.js.map +1 -1
  129. package/local/daemon/launch.d.ts +12 -4
  130. package/local/daemon/launch.d.ts.map +1 -1
  131. package/local/daemon/launch.js +49 -16
  132. package/local/daemon/launch.js.map +1 -1
  133. package/local/daemon/types.d.ts +9 -1
  134. package/local/daemon/types.d.ts.map +1 -1
  135. package/local/key-check.d.ts +20 -0
  136. package/local/key-check.d.ts.map +1 -0
  137. package/local/key-check.js +45 -0
  138. package/local/key-check.js.map +1 -0
  139. package/local/model-key-step.d.ts +63 -0
  140. package/local/model-key-step.d.ts.map +1 -0
  141. package/local/model-key-step.js +133 -0
  142. package/local/model-key-step.js.map +1 -0
  143. package/local/setup/prompt.d.ts.map +1 -1
  144. package/local/setup/prompt.js +22 -18
  145. package/local/setup/prompt.js.map +1 -1
  146. package/local/setup/wizard.d.ts +10 -0
  147. package/local/setup/wizard.d.ts.map +1 -1
  148. package/local/setup/wizard.js +40 -17
  149. package/local/setup/wizard.js.map +1 -1
  150. package/local/state/lock.d.ts.map +1 -1
  151. package/local/state/lock.js +123 -36
  152. package/local/state/lock.js.map +1 -1
  153. package/local/state/pidfile.d.ts +4 -0
  154. package/local/state/pidfile.d.ts.map +1 -1
  155. package/local/state/pidfile.js +12 -3
  156. package/local/state/pidfile.js.map +1 -1
  157. package/package.json +12 -9
  158. package/program.d.ts.map +1 -1
  159. package/program.js +10 -0
  160. package/program.js.map +1 -1
  161. package/resources/get.js +1 -1
  162. package/resources/get.js.map +1 -1
  163. package/resources/run/stream.d.ts.map +1 -1
  164. package/resources/run/stream.js +2 -1
  165. package/resources/run/stream.js.map +1 -1
  166. package/resources/stream/render-plaintext.d.ts +3 -0
  167. package/resources/stream/render-plaintext.d.ts.map +1 -1
  168. package/resources/stream/render-plaintext.js +20 -2
  169. package/resources/stream/render-plaintext.js.map +1 -1
  170. package/resources/usage.d.ts +8 -0
  171. package/resources/usage.d.ts.map +1 -1
  172. package/resources/usage.js +37 -0
  173. package/resources/usage.js.map +1 -1
  174. package/runtime.d.ts +9 -0
  175. package/runtime.d.ts.map +1 -1
  176. package/runtime.js +12 -1
  177. package/runtime.js.map +1 -1
  178. package/src/auth/browser.ts +57 -0
  179. package/src/auth/callback-server.ts +58 -22
  180. package/src/auth/device.ts +151 -0
  181. package/src/auth/index.ts +4 -3
  182. package/src/auth/login.ts +243 -67
  183. package/src/auth/oidc.ts +270 -0
  184. package/src/auth/token.ts +53 -90
  185. package/src/client/client.ts +4 -1
  186. package/src/client/current-organization.ts +27 -0
  187. package/src/commands/apikey/index.ts +85 -0
  188. package/src/commands/app-handover.ts +188 -0
  189. package/src/commands/auth/index.ts +16 -10
  190. package/src/commands/auth/login.ts +30 -23
  191. package/src/commands/auth/whoami.ts +16 -1
  192. package/src/commands/coding-agent.ts +33 -0
  193. package/src/commands/config/backend.ts +20 -6
  194. package/src/commands/embedding.ts +209 -0
  195. package/src/commands/example.ts +146 -0
  196. package/src/commands/help.ts +16 -3
  197. package/src/commands/load-test/burst.ts +291 -0
  198. package/src/commands/load-test/index.ts +183 -0
  199. package/src/commands/load-test/report.ts +302 -0
  200. package/src/commands/mcp-server.ts +5 -5
  201. package/src/commands/setup.ts +76 -2
  202. package/src/commands/up.ts +76 -10
  203. package/src/commands/usage.ts +55 -2
  204. package/src/config/config.ts +18 -8
  205. package/src/config/index.ts +1 -0
  206. package/src/config/resolve.ts +5 -3
  207. package/src/errors/classify.ts +10 -1
  208. package/src/errors/cli-exit-error.ts +6 -2
  209. package/src/errors/handle.ts +74 -8
  210. package/src/local/daemon/components.ts +66 -2
  211. package/src/local/daemon/host.ts +6 -2
  212. package/src/local/daemon/launch.ts +51 -16
  213. package/src/local/daemon/types.ts +9 -1
  214. package/src/local/key-check.ts +61 -0
  215. package/src/local/model-key-step.ts +164 -0
  216. package/src/local/setup/prompt.ts +22 -18
  217. package/src/local/setup/wizard.ts +45 -18
  218. package/src/local/state/lock.ts +116 -35
  219. package/src/local/state/pidfile.ts +14 -3
  220. package/src/program.ts +10 -0
  221. package/src/resources/get.ts +1 -1
  222. package/src/resources/run/stream.ts +2 -1
  223. package/src/resources/stream/render-plaintext.ts +20 -2
  224. package/src/resources/usage.ts +42 -0
  225. package/src/runtime.ts +17 -1
  226. package/auth/auth0.d.ts +0 -19
  227. package/auth/auth0.d.ts.map +0 -1
  228. package/auth/auth0.js +0 -39
  229. package/auth/auth0.js.map +0 -1
  230. package/src/auth/auth0.ts +0 -43
@@ -29,6 +29,9 @@ export const COMMAND_GROUPS: Readonly<Record<string, HelpGroup>> = {
29
29
  run: "Run agents",
30
30
  resume: "Run agents",
31
31
  runs: "Run agents",
32
+ "app-handover": "Run agents",
33
+ "coding-agent": "Run agents",
34
+ "load-test": "Run agents",
32
35
  get: "Your library",
33
36
  list: "Your library",
34
37
  search: "Your library",
@@ -42,6 +45,7 @@ export const COMMAND_GROUPS: Readonly<Record<string, HelpGroup>> = {
42
45
  vault: "Your library",
43
46
  file: "Your library",
44
47
  connect: "Your library",
48
+ example: "Your library",
45
49
  push: "Packages",
46
50
  install: "Packages",
47
51
  marketplace: "Packages",
@@ -58,6 +62,7 @@ export const COMMAND_GROUPS: Readonly<Record<string, HelpGroup>> = {
58
62
  auth: "Account and access",
59
63
  apikey: "Account and access",
60
64
  "service-account": "Account and access",
65
+ embedding: "Account and access",
61
66
  "provider-key": "Account and access",
62
67
  "access-history": "Account and access",
63
68
  usage: "Account and access",
@@ -71,15 +76,17 @@ export const HELP_EXAMPLES: ReadonlyMap<string, readonly HelpExample[]> = new Ma
71
76
  ["stigmer access-history export", [{ say: "everything, oldest first, one JSON object per line", run: "stigmer access-history export > access-history.jsonl" }, { say: "only what came after the last entry you hold", run: "stigmer access-history export --after 42 >> access-history.jsonl" }]],
72
77
  ["stigmer access-history list", [{ say: "the newest 50 changes", run: "stigmer access-history list" }, { say: "the newest 200, as JSON", run: "stigmer access-history list --limit 200 -o json" }]],
73
78
  ["stigmer agent quality", [{ say: "how the agent's runs were graded over the last 30 days, per version", run: "stigmer agent quality variance-reporter" }, { say: "the last week's failed grades, newest first", run: "stigmer agent quality variance-reporter --days 7 --flagged --limit 50" }]],
74
- ["stigmer apikey create", [{ say: "create a new API key", run: "stigmer apikey create --name \"ci-pipeline\"" }, { say: "create a key that never expires", run: "stigmer apikey create --name \"automation\" --never-expires" }]],
79
+ ["stigmer apikey create", [{ say: "create a new API key", run: "stigmer apikey create --name \"ci-pipeline\"" }, { say: "create a key that never expires", run: "stigmer apikey create --name \"automation\" --never-expires" }, { say: "a key for your app's backend, saved to its env file and never printed", run: "stigmer apikey create --service-account app-backend --name my-app --save-to-env .env.local" }]],
80
+ ["stigmer app-handover", [{ say: "print the token route and chat for a Next.js app", run: "stigmer app-handover --stack nextjs --agent invoice-matcher" }, { say: "write them into the app you are in", run: "stigmer app-handover --stack node --agent invoice-matcher --write ." }]],
75
81
  ["stigmer apikey fingerprint", [{ say: "look up which key matches a raw token", run: "stigmer apikey fingerprint stk_abc123..." }]],
76
82
  ["stigmer apply", [{ say: "apply a single resource file", run: "stigmer apply -f agent.yaml" }, { say: "apply all resources in a directory", run: "stigmer apply -f ./manifests/" }]],
77
- ["stigmer auth login", [{ say: "log in to Stigmer Cloud", run: "stigmer auth login" }]],
83
+ ["stigmer auth login", [{ say: "sign in to the active backend's server in your browser", run: "stigmer auth login" }, { say: "where no browser opens here, sign in with a code on any device", run: "stigmer auth login --device" }]],
78
84
  ["stigmer auth logout", [{ say: "log out", run: "stigmer auth logout" }]],
79
85
  ["stigmer auth whoami", [{ say: "check who you're logged in as", run: "stigmer auth whoami" }]],
80
86
  ["stigmer channel disable", [{ say: "pause any channel (web, Slack or WhatsApp), keeping its configuration", run: "stigmer channel disable acme-corp/variance-reporter-slack" }]],
81
87
  ["stigmer channel enable", [{ say: "turn it back on", run: "stigmer channel enable acme-corp/variance-reporter-slack" }]],
82
88
  ["stigmer channel link", [{ say: "turn on an Agent's chat link for your Organization's members", run: "stigmer channel link variance-reporter" }, { say: "make the link public and print the embed snippet", run: "stigmer channel link variance-reporter --audience public" }]],
89
+ ["stigmer coding-agent", [{ say: "print the guide your coding agent follows", run: "stigmer coding-agent" }]],
83
90
  ["stigmer completion", [{ say: "install permanently", run: "stigmer completion bash > /etc/bash_completion.d/stigmer" }]],
84
91
  ["stigmer config backend add", [{ say: "register a self-hosted server (mint the key with `stigmer apikey create`)", run: "stigmer config backend add staging --endpoint stigmer.example.com:7234 --api-key stk_..." }]],
85
92
  ["stigmer config backend list", [{ say: "see what is configured", run: "stigmer config backend list" }]],
@@ -98,12 +105,17 @@ export const HELP_EXAMPLES: ReadonlyMap<string, readonly HelpExample[]> = new Ma
98
105
  ["stigmer down", [{ say: "stop the local stack", run: "stigmer down" }]],
99
106
  ["stigmer down server", [{ say: "stop the whole stack, as stigmer down does", run: "stigmer down server" }]],
100
107
  ["stigmer download", [{ say: "download all outputs from a run", run: "stigmer download run run_01abc123" }, { say: "download to a specific directory", run: "stigmer download run run_01abc123 --output-dir ./results/" }]],
108
+ ["stigmer embedding", [{ say: "how people sign in to your organization from your app", run: "stigmer embedding" }]],
109
+ ["stigmer embedding set", [{ say: "make an app user's account on their first sign-in, as a Member", run: "stigmer embedding set --create-accounts-on-sign-in on --sign-in-role member" }]],
101
110
  ["stigmer eval cancel", [{ say: "cancel a running eval run", run: "stigmer eval cancel evr_01abc123" }]],
102
111
  ["stigmer eval run", [{ say: "a plugin's own cases on the built-in assistant, with and without it", run: "stigmer eval run --plugin thermos" }, { say: "the cases of every plugin the agent attaches, on the agent", run: "stigmer eval run --agent variance-reporter" }, { say: "the cases of an Eval pushed with stigmer push eval, at its stable version", run: "stigmer eval run --agent variance-reporter --eval variance-reporter-evals@stable" }, { say: "the plugin's cases on the agent, with and without the plugin", run: "stigmer eval run --agent variance-reporter --plugin payroll" }]],
112
+ ["stigmer example get", [{ say: "write an example's manifest and samples into stigmer/invoice-match", run: "stigmer example get invoice-match" }, { say: "write a ready-made skill to push", run: "stigmer example get spreadsheet-export" }]],
113
+ ["stigmer example list", [{ say: "see the example agents and skills", run: "stigmer example list" }]],
103
114
  ["stigmer file upload", [{ say: "upload a spreadsheet to reuse, and see the id it is named by", run: "stigmer file upload q3-ledger.xlsx" }, { say: "upload into a conversation you already have", run: "stigmer file upload notes.md --session ses_01abc123" }]],
104
115
  ["stigmer get", [{ say: "get an agent by slug", run: "stigmer get agent my-agent" }, { say: "get by org-qualified slug", run: "stigmer get agent acme-corp/variance-reporter" }]],
105
116
  ["stigmer install", [{ say: "install from the official catalogue by name", run: "stigmer install linear" }, { say: "install from a source you added, naming it", run: "stigmer install acme-plugins/thermos" }]],
106
117
  ["stigmer list", [{ say: "list all agents", run: "stigmer list agents" }, { say: "list available resource types", run: "stigmer list types" }]],
118
+ ["stigmer load-test", [{ say: "start 25 agents at once on the active server", run: "stigmer load-test --agents 25" }, { say: "a bigger burst on a cheaper model, as JSON for a script", run: "stigmer load-test --agents 100 --model claude-haiku-4-5 --max-usd 2 -o json > burst.json" }]],
107
119
  ["stigmer logs", [{ say: "follow all components live (tail snapshot first)", run: "stigmer logs" }, { say: "show the last 100 lines and exit", run: "stigmer logs --no-follow --tail 100" }]],
108
120
  ["stigmer marketplace add", [{ say: "add your company's catalogue (a public GitHub repository with a marketplace file)", run: "stigmer marketplace add acme/agent-plugins" }, { say: "add a repository at a branch or tag, or from its URL", run: "stigmer marketplace add acme/agent-plugins@main" }]],
109
121
  ["stigmer marketplace list", [{ say: "see what is configured, and what one offers", run: "stigmer marketplace list" }]],
@@ -135,7 +147,7 @@ export const HELP_EXAMPLES: ReadonlyMap<string, readonly HelpExample[]> = new Ma
135
147
  ["stigmer service-account delete", [{ say: "delete one; every key it has stops working at once", run: "stigmer service-account delete ci-deploy" }]],
136
148
  ["stigmer service-account keys", [{ say: "list one's API keys; revoke one with: stigmer delete apikey <id>", run: "stigmer service-account keys ci-deploy" }]],
137
149
  ["stigmer service-account list", [{ say: "list the Organization's service accounts, newest first", run: "stigmer service-account list" }]],
138
- ["stigmer setup", [{ say: "interactive setup", run: "stigmer setup" }, { say: "non-interactive: configure Anthropic", run: "stigmer setup --provider anthropic --api-key sk-ant-xxx" }]],
150
+ ["stigmer setup", [{ say: "interactive setup", run: "stigmer setup" }, { say: "non-interactive: configure Anthropic", run: "stigmer setup --provider anthropic --api-key sk-ant-xxx" }, { say: "the key through stdin, kept out of history and transcripts", run: "stigmer setup --api-key-stdin < anthropic.key" }]],
139
151
  ["stigmer status", [{ say: "show component health and LLM configuration", run: "stigmer status" }, { say: "machine-readable status for scripts", run: "stigmer status --json" }]],
140
152
  ["stigmer tag", [{ say: "tag an agent version as stable, by the short hash the history prints", run: "stigmer tag agent acme-corp/variance-reporter 3f8a1c0e9b7d stable" }, { say: "the same, by its full 64-character hash", run: "stigmer tag agent acme-corp/variance-reporter 3f8a1c0e9b7d2a4c6e5f1b3d9a7c0e2f4b6d8a1c3e5f7b9d0a2c4e6f8b1d3a5c stable" }]],
141
153
  ["stigmer up", [{ say: "start the full local stack (server, runner, Temporal)", run: "stigmer up" }]],
@@ -143,6 +155,7 @@ export const HELP_EXAMPLES: ReadonlyMap<string, readonly HelpExample[]> = new Ma
143
155
  ["stigmer usage agent", [{ say: "view agent usage for a date range", run: "stigmer usage agent agt-01abc123 --from 2026-03-01 --to 2026-03-31" }]],
144
156
  ["stigmer usage hours", [{ say: "view the organization's agent hours (Stigmer Cloud)", run: "stigmer usage hours" }]],
145
157
  ["stigmer usage org", [{ say: "view organization-wide usage", run: "stigmer usage org --from 2026-03-01 --to 2026-03-31" }]],
158
+ ["stigmer usage onboarding", [{ say: "see where last week's new organizations stopped (platform operators)", run: "stigmer usage onboarding" }]],
146
159
  ["stigmer usage session", [{ say: "view usage for a specific session", run: "stigmer usage session ses_01j9z3k8m2q4r6t8v0w2y4a6c8" }]],
147
160
  ["stigmer validate", [{ say: "validate a single file", run: "stigmer validate -f agent.yaml" }, { say: "validate all YAML files in a directory", run: "stigmer validate -f ./manifests/" }]],
148
161
  ["stigmer vault connect-link", [{ say: "make a one-time sign-in page whose login lands in a shared vault", run: "stigmer vault connect-link ledger-keys --address https://mcp.linear.app/mcp --return-url https://app.example.com/settings" }]],
@@ -0,0 +1,291 @@
1
+ // The engine of `stigmer load-test`: make one agent of its own, start N
2
+ // turns on it at the same moment, watch each until it ends or the watch's
3
+ // time is up, read each run once, then delete everything it made.
4
+ //
5
+ // - Every turn is a new conversation (`startTurn` with a `sessionSpec`), so
6
+ // each asks the fleet for a sandbox of its own. The creates are fired
7
+ // together (one `Promise.allSettled`, no stagger); a create the server
8
+ // refuses is kept as its code and the server's own sentence.
9
+ // - Each admitted turn is followed the way `stigmer run` follows one
10
+ // (`followSession` with the `stigmer.run.liveness` preview), never with
11
+ // `waitForTurn`, which asks for no previews and so never sees a wait. The
12
+ // largest `awaiting_sandbox_for` any frame carried is kept: a later
13
+ // liveness replaces the fold's, so the fold forgets the wait once the turn
14
+ // starts.
15
+ // - A started turn's conversation is left as a person leaves one until the
16
+ // cleanup at the end: its sandbox holds its place until the fleet's own
17
+ // idle sweep, which is how room frees for the turns still waiting.
18
+ // - The cleanup runs on every way out, Ctrl-C included: every follow is
19
+ // aborted, every run still open is terminated, every conversation it made
20
+ // is deleted (with its sandbox), then the agent. What could not be deleted
21
+ // is returned by id. This command cannot read a server's database, so it
22
+ // never claims the fleet is empty.
23
+ //
24
+ // The client is the narrow slice of `Stigmer` the engine calls, so tests
25
+ // drive it with a fake.
26
+
27
+ import type { Agent } from "@stigmer/apis/ai/stigmer/agentic/agent/v1/api_pb";
28
+ import type { Run } from "@stigmer/apis/ai/stigmer/agentic/run/v1/api_pb";
29
+ import { RunPhase } from "@stigmer/apis/ai/stigmer/agentic/run/v1/enum_pb";
30
+ import type { Duration } from "@bufbuild/protobuf/wkt";
31
+ import {
32
+ type AgentInput,
33
+ followSession,
34
+ isTerminalPhase,
35
+ type RunInput,
36
+ type SessionEventsClient,
37
+ SessionLog,
38
+ startTurn,
39
+ StigmerError,
40
+ turnOf,
41
+ type TurnPhase,
42
+ } from "@stigmer/client";
43
+ import type { Refusal, RunSighting } from "./report.js";
44
+
45
+ /** The slice of the client the engine calls. */
46
+ export interface LoadTestClient extends SessionEventsClient {
47
+ readonly agent: {
48
+ create(input: AgentInput): Promise<Agent>;
49
+ delete(id: string): Promise<unknown>;
50
+ };
51
+ readonly run: {
52
+ create(input: RunInput): Promise<Run>;
53
+ get(id: string): Promise<Run>;
54
+ terminate(input: { id: string; reason: string }): Promise<unknown>;
55
+ };
56
+ readonly session: SessionEventsClient["session"] & {
57
+ delete(id: string): Promise<unknown>;
58
+ };
59
+ }
60
+
61
+ /** How the burst stands, each time it changes. */
62
+ export interface BurstProgress {
63
+ readonly sent: number;
64
+ readonly started: number;
65
+ /** Admitted and not yet started: waiting for a sandbox, or still setting up. */
66
+ readonly waiting: number;
67
+ readonly refused: number;
68
+ }
69
+
70
+ export interface BurstOptions {
71
+ /** The organization the agent and the turns are made in ("" lets a single-organization server fill it). */
72
+ readonly org: string;
73
+ readonly agents: number;
74
+ /** The model every turn asks for; "" leaves the agent's default. */
75
+ readonly model: string;
76
+ readonly message: string;
77
+ /** Each turn's budget, `RunConfig.max_cost_usd`. */
78
+ readonly shareUsd: number;
79
+ /** How long to watch, from the burst, before calling a run still waiting. */
80
+ readonly waitMs: number;
81
+ /** Ctrl-C: stops every watch; the cleanup still runs. */
82
+ readonly signal: AbortSignal;
83
+ readonly onProgress?: (progress: BurstProgress) => void;
84
+ /** The agent's name suffix; random when absent (tests pin it). */
85
+ readonly nonce?: string;
86
+ }
87
+
88
+ /** What the burst saw, and what its cleanup could not delete. */
89
+ export interface BurstResult {
90
+ /** The agent's slug. */
91
+ readonly agent: string;
92
+ readonly sightings: readonly RunSighting[];
93
+ /** "<kind> <id>" for each thing the cleanup could not delete. */
94
+ readonly leftBehind: readonly string[];
95
+ /** Whether Ctrl-C ended the watch. */
96
+ readonly interrupted: boolean;
97
+ }
98
+
99
+ /** The one-line instructions the burst's agent runs: each turn stays tiny. */
100
+ export const AGENT_INSTRUCTIONS = "Reply with one word.";
101
+
102
+ const TERMINATE_REASON = "stigmer load-test cleanup";
103
+
104
+ // The fold's phases a turn has only once its runner started it.
105
+ const STARTED_PHASES: ReadonlySet<TurnPhase> = new Set<TurnPhase>(["running", "waiting", "paused", "completed"]);
106
+
107
+ interface Admitted {
108
+ readonly index: number;
109
+ readonly runId: string;
110
+ readonly sessionId: string;
111
+ }
112
+
113
+ /** Run the burst (module header). The agent's create throws through; everything after it is cleaned up. */
114
+ export async function runBurst(client: LoadTestClient, options: BurstOptions): Promise<BurstResult> {
115
+ const nonce = options.nonce ?? Math.random().toString(36).slice(2, 8);
116
+ const agent = await client.agent.create({
117
+ name: `load-test-${nonce}`,
118
+ org: options.org,
119
+ description: "Made by stigmer load-test and deleted when it ends.",
120
+ instructions: AGENT_INSTRUCTIONS,
121
+ });
122
+ const agentId = agent.metadata?.id ?? "";
123
+ const agentRef = { org: agent.metadata?.org ?? options.org, slug: agent.metadata?.slug ?? "" };
124
+
125
+ const deadline = new AbortController();
126
+ const timer = setTimeout(() => deadline.abort(), options.waitMs);
127
+ const signal = AbortSignal.any([options.signal, deadline.signal]);
128
+ const tracker = new Tracker(options.agents, options.onProgress);
129
+ const slots: Array<Admitted | Refusal> = [];
130
+ const admitted: Admitted[] = [];
131
+ const waits = new Map<string, number>();
132
+ const runs = new Map<string, Run>();
133
+ try {
134
+ const sends = await Promise.allSettled(
135
+ Array.from({ length: options.agents }, () =>
136
+ startTurn(client, {
137
+ org: agentRef.org,
138
+ message: options.message,
139
+ sessionSpec: { agentRef, includeMyVault: true },
140
+ runConfig: { maxCostUsd: options.shareUsd, ...(options.model === "" ? {} : { modelName: options.model }) },
141
+ }),
142
+ ),
143
+ );
144
+ for (const [i, send] of sends.entries()) {
145
+ if (send.status === "fulfilled") {
146
+ const a = { index: i + 1, runId: send.value.runId, sessionId: send.value.sessionId };
147
+ admitted.push(a);
148
+ slots.push(a);
149
+ } else {
150
+ slots.push(refusalOf(send.reason));
151
+ }
152
+ }
153
+ tracker.sent(admitted.map((a) => a.runId), slots.length - admitted.length);
154
+
155
+ await Promise.all(
156
+ admitted.map(async (a) => {
157
+ const waited = await watch(client, a, signal, tracker);
158
+ if (waited !== undefined) waits.set(a.runId, waited);
159
+ }),
160
+ );
161
+ await Promise.all(
162
+ admitted.map(async (a) => {
163
+ const run = await client.run.get(a.runId).catch(() => undefined);
164
+ if (run !== undefined) runs.set(a.runId, run);
165
+ }),
166
+ );
167
+ } finally {
168
+ clearTimeout(timer);
169
+ }
170
+
171
+ const leftBehind = await cleanUp(client, admitted, runs, agentId);
172
+ const sightings = slots.map((slot, i): RunSighting =>
173
+ "runId" in slot
174
+ ? { index: slot.index, runId: slot.runId, sessionId: slot.sessionId, run: runs.get(slot.runId), waitedMs: waits.get(slot.runId) }
175
+ : { index: i + 1, runId: "", sessionId: "", refusal: slot },
176
+ );
177
+ return { agent: agentRef.slug, sightings, leftBehind, interrupted: options.signal.aborted };
178
+ }
179
+
180
+ /** A refused create as its code word and the server's own sentence. */
181
+ export function refusalOf(err: unknown): Refusal {
182
+ if (err instanceof StigmerError) return { code: err.code, message: err.message };
183
+ return { code: "unknown", message: err instanceof Error ? err.message : String(err) };
184
+ }
185
+
186
+ // Follow one turn until it ends or the watch stops; the largest wait for a
187
+ // sandbox its liveness carried, or undefined when it reported none. A follow
188
+ // the server refuses ends the watch; the run is read afterwards either way.
189
+ async function watch(client: LoadTestClient, a: Admitted, signal: AbortSignal, tracker: Tracker): Promise<number | undefined> {
190
+ const own = new AbortController();
191
+ const log = new SessionLog();
192
+ let waited: number | undefined;
193
+ try {
194
+ for await (const frame of followSession(client, a.sessionId, {
195
+ signal: AbortSignal.any([signal, own.signal]),
196
+ previews: ["stigmer.run.liveness"],
197
+ })) {
198
+ if (frame.frame.case === "stigmerRunLiveness" && frame.frame.value.runId === a.runId) {
199
+ const awaiting = frame.frame.value.awaitingSandboxFor;
200
+ if (awaiting !== undefined) waited = Math.max(waited ?? 0, durationMs(awaiting));
201
+ }
202
+ if (!log.applyFrame(frame)) continue;
203
+ const turn = turnOf(log.snapshot(), a.runId);
204
+ if (turn?.status === undefined) continue;
205
+ if (STARTED_PHASES.has(turn.status.phase)) tracker.started(a.runId);
206
+ if (turn.ended) {
207
+ tracker.ended(a.runId);
208
+ break;
209
+ }
210
+ }
211
+ } catch {
212
+ // A refused follow: the run's own read says how it stands.
213
+ } finally {
214
+ own.abort();
215
+ }
216
+ return waited;
217
+ }
218
+
219
+ // Terminate what is still open, delete each conversation, then the agent;
220
+ // return what could not be deleted.
221
+ async function cleanUp(client: LoadTestClient, admitted: readonly Admitted[], runs: ReadonlyMap<string, Run>, agentId: string): Promise<string[]> {
222
+ const left: string[] = [];
223
+ await Promise.all(
224
+ admitted
225
+ .filter((a) => !isTerminalPhase(runs.get(a.runId)?.status?.phase ?? RunPhase.RUN_PHASE_UNSPECIFIED))
226
+ .map((a) => client.run.terminate({ id: a.runId, reason: TERMINATE_REASON }).catch(() => undefined)),
227
+ );
228
+ await Promise.all(
229
+ admitted.map(async (a) => {
230
+ try {
231
+ await client.session.delete(a.sessionId);
232
+ } catch {
233
+ left.push(`conversation ${a.sessionId}`);
234
+ }
235
+ }),
236
+ );
237
+ if (agentId !== "") {
238
+ try {
239
+ await client.agent.delete(agentId);
240
+ } catch {
241
+ left.push(`agent ${agentId}`);
242
+ }
243
+ }
244
+ return left;
245
+ }
246
+
247
+ function durationMs(duration: Duration): number {
248
+ return Number(duration.seconds) * 1000 + Math.floor(duration.nanos / 1_000_000);
249
+ }
250
+
251
+ // The burst's counts, told to the caller each time one changes.
252
+ class Tracker {
253
+ private readonly startedRuns = new Set<string>();
254
+ private readonly endedRuns = new Set<string>();
255
+ private admitted: readonly string[] = [];
256
+ private refused = 0;
257
+ private lastTold = "";
258
+
259
+ constructor(
260
+ private readonly total: number,
261
+ private readonly onProgress: ((progress: BurstProgress) => void) | undefined,
262
+ ) {}
263
+
264
+ sent(admitted: readonly string[], refused: number): void {
265
+ this.admitted = admitted;
266
+ this.refused = refused;
267
+ this.tell();
268
+ }
269
+
270
+ started(runId: string): void {
271
+ if (this.startedRuns.has(runId)) return;
272
+ this.startedRuns.add(runId);
273
+ this.tell();
274
+ }
275
+
276
+ // Once per run: its watch stops at its end.
277
+ ended(runId: string): void {
278
+ this.endedRuns.add(runId);
279
+ this.tell();
280
+ }
281
+
282
+ // Told only when a count moved: a run that ends after it started changes none.
283
+ private tell(): void {
284
+ const waiting = this.admitted.filter((id) => !this.startedRuns.has(id) && !this.endedRuns.has(id)).length;
285
+ const progress = { sent: this.total, started: this.startedRuns.size, waiting, refused: this.refused };
286
+ const key = JSON.stringify(progress);
287
+ if (key === this.lastTold) return;
288
+ this.lastTold = key;
289
+ this.onProgress?.(progress);
290
+ }
291
+ }
@@ -0,0 +1,183 @@
1
+ // `stigmer load-test --agents <n>`: start n agents at the same moment on the
2
+ // connected server and report how many started, how long each waited for a
3
+ // sandbox, and which were refused and why. A top-level verb in "Run agents".
4
+ //
5
+ // Thin handler: check the flags, resolve the server and the organization,
6
+ // say the budget before spending it, run the burst (burst.ts), print the
7
+ // report (report.ts). The CLI's laws hold: the report is data on stdout
8
+ // (`-o table|json|yaml`), the budget, the progress and the cleanup are on
9
+ // stderr, and there is no prompt (it destroys nothing of the person's;
10
+ // `stigmer run` spends without one).
11
+ //
12
+ // The budget is the whole burst's (`--max-usd`, $1 by default), split evenly
13
+ // into each run's `max_cost_usd`; a run stops once its estimate reaches its
14
+ // share, and one model call can go past it. There is no cap on n: the budget
15
+ // is the guard, and a number the tool refused would be a fleet size of its
16
+ // own.
17
+ //
18
+ // Exit codes are the CLI's table: 0 when the burst ran and was reported,
19
+ // whatever it found (the report is the verdict); a refusal before the burst
20
+ // (the key, the organization, the agent's create) exits with its own code;
21
+ // 1 when the cleanup left something, listed on stderr; 130 after Ctrl-C,
22
+ // once the cleanup ran.
23
+
24
+ import type { Command } from "commander";
25
+ import { ensureAuthenticated, resolveOrganization } from "../../config/index.js";
26
+ import { CliExitError, ExitCode, UsageError } from "../../errors/index.js";
27
+ import type { OutputFlags } from "../../output/index.js";
28
+ import { dialedServerAddress } from "../../runtime.js";
29
+ import { globalOrg, readFormat } from "../shared.js";
30
+ import type { BurstProgress } from "./burst.js";
31
+
32
+ interface LoadTestFlags extends OutputFlags {
33
+ agents?: string;
34
+ model?: string;
35
+ message?: string;
36
+ maxUsd?: string;
37
+ waitMinutes?: string;
38
+ }
39
+
40
+ /** What each agent is asked when `--message` is not given. */
41
+ export const DEFAULT_MESSAGE = "Say hello in one word.";
42
+
43
+ const DEFAULT_MAX_USD = 1;
44
+ const DEFAULT_WAIT_MINUTES = 20;
45
+
46
+ export function registerLoadTest(program: Command): void {
47
+ program
48
+ .command("load-test")
49
+ .description("start many agents at once and report how many started, how long each waited, and which were refused")
50
+ .option("--agents <n>", "how many agents to start at the same moment (required)")
51
+ .option("--model <id>", "the model each run uses; the agent's default when unset")
52
+ .option("--message <text>", `what each agent is asked (default "${DEFAULT_MESSAGE}")`)
53
+ .option("--max-usd <total>", `the whole burst's budget in US dollars, split evenly across the runs (default ${DEFAULT_MAX_USD})`)
54
+ .option("--wait-minutes <m>", `how long to watch for runs to start before calling them still waiting (default ${DEFAULT_WAIT_MINUTES})`)
55
+ .option("-o, --output <format>", "output format: table, json, or yaml")
56
+ .action((options: LoadTestFlags, command: Command) => runLoadTest(options, command));
57
+ }
58
+
59
+ /** The flags, checked: a count of at least one, a positive budget and watch. */
60
+ export function parseFlags(options: LoadTestFlags): { agents: number; maxUsd: number; waitMinutes: number } {
61
+ const agents = Number(options.agents ?? "");
62
+ if (!Number.isInteger(agents) || agents < 1) {
63
+ throw new UsageError("--agents takes a whole number of at least 1");
64
+ }
65
+ const maxUsd = options.maxUsd === undefined ? DEFAULT_MAX_USD : Number(options.maxUsd);
66
+ if (!Number.isFinite(maxUsd) || maxUsd <= 0) {
67
+ throw new UsageError("--max-usd takes an amount above 0, in US dollars");
68
+ }
69
+ const waitMinutes = options.waitMinutes === undefined ? DEFAULT_WAIT_MINUTES : Number(options.waitMinutes);
70
+ if (!Number.isFinite(waitMinutes) || waitMinutes <= 0) {
71
+ throw new UsageError("--wait-minutes takes a number of minutes above 0");
72
+ }
73
+ return { agents, maxUsd, waitMinutes };
74
+ }
75
+
76
+ async function runLoadTest(options: LoadTestFlags, command: Command): Promise<void> {
77
+ const { agents, maxUsd, waitMinutes } = parseFlags(options);
78
+ const format = readFormat(options);
79
+ const [{ connectBackend }, { runBurst }, report, { formatElapsed }] = await Promise.all([
80
+ import("../../backend.js"),
81
+ import("./burst.js"),
82
+ import("./report.js"),
83
+ import("@stigmer/client"),
84
+ ]);
85
+ const client = connectBackend();
86
+ ensureAuthenticated(client.config);
87
+ const org = resolveOrganization(client.config, globalOrg(command));
88
+ const server = dialedServerAddress()?.address ?? "";
89
+ const shareUsd = maxUsd / agents;
90
+
91
+ const where = [org === "" ? "" : ` in ${org}`, server === "" ? "" : ` on ${server}`].join("");
92
+ process.stderr.write(
93
+ `Starting ${agents} agent${agents === 1 ? "" : "s"} at once${where}.\n` +
94
+ `Budget ${report.usd(maxUsd)}, split across the runs: each stops once it has spent ${report.usd(shareUsd)}. One model call can go past a run's share.\n`,
95
+ );
96
+
97
+ const controller = new AbortController();
98
+ const onSignal = (): void => controller.abort();
99
+ process.once("SIGINT", onSignal);
100
+ process.once("SIGTERM", onSignal);
101
+ const progress = progressPrinter(process.stderr, formatElapsed);
102
+ try {
103
+ const result = await runBurst(client.stigmer, {
104
+ org,
105
+ agents,
106
+ model: options.model ?? "",
107
+ message: options.message ?? DEFAULT_MESSAGE,
108
+ shareUsd,
109
+ waitMs: waitMinutes * 60_000,
110
+ signal: controller.signal,
111
+ onProgress: progress.update,
112
+ });
113
+ progress.stop();
114
+ process.stderr.write(cleanupLine(result.sightings.filter((s) => s.sessionId !== "").length, result.leftBehind));
115
+ const built = report.buildReport(
116
+ { server, org, agent: result.agent, requested: agents, budgetUsd: maxUsd, shareUsd },
117
+ result.sightings,
118
+ result.leftBehind,
119
+ );
120
+ process.stdout.write(report.renderReport(built, format));
121
+ if (result.interrupted) throw new CliExitError("", ExitCode.Interrupted);
122
+ if (result.leftBehind.length > 0) throw new CliExitError("", ExitCode.General);
123
+ } finally {
124
+ progress.stop();
125
+ process.removeListener("SIGINT", onSignal);
126
+ process.removeListener("SIGTERM", onSignal);
127
+ }
128
+ }
129
+
130
+ /** The cleanup's line: what was deleted, or what is left to remove by hand. */
131
+ export function cleanupLine(conversations: number, leftBehind: readonly string[]): string {
132
+ if (leftBehind.length > 0) return `Could not delete, remove by hand: ${leftBehind.join(", ")}\n`;
133
+ const deleted = conversations === 0 ? "the agent" : `${conversations} conversation${conversations === 1 ? "" : "s"} and the agent`;
134
+ return `Cleaned up: deleted ${deleted}.\n`;
135
+ }
136
+
137
+ /** The stream the progress line is written to. */
138
+ export interface ProgressStream {
139
+ readonly isTTY?: boolean;
140
+ write(text: string): boolean;
141
+ }
142
+
143
+ /**
144
+ * The progress line, "6 sent · 3 started · 2 waiting · 1 refused · 42s": on
145
+ * a terminal one line rewritten in place, its time ticking each second; off
146
+ * one, a line per change. `stop` ends the line; calling it again does nothing.
147
+ */
148
+ export function progressPrinter(
149
+ stream: ProgressStream,
150
+ elapsed: (ms: number) => string,
151
+ now: () => number = Date.now,
152
+ ): { update: (progress: BurstProgress) => void; stop: () => void } {
153
+ const begin = now();
154
+ let last: BurstProgress | undefined;
155
+ let ticker: ReturnType<typeof setInterval> | undefined;
156
+ const text = (p: BurstProgress): string => {
157
+ const parts = [`${p.sent} sent`, `${p.started} started`, `${p.waiting} waiting`];
158
+ if (p.refused > 0) parts.push(`${p.refused} refused`);
159
+ parts.push(elapsed(now() - begin));
160
+ return parts.join(" · ");
161
+ };
162
+ const draw = (): void => {
163
+ if (last !== undefined) stream.write(`\r\u001b[2K${text(last)}`);
164
+ };
165
+ return {
166
+ update(progress) {
167
+ last = progress;
168
+ if (stream.isTTY !== true) {
169
+ stream.write(`${text(progress)}\n`);
170
+ return;
171
+ }
172
+ draw();
173
+ ticker ??= setInterval(draw, 1000);
174
+ ticker.unref();
175
+ },
176
+ stop() {
177
+ if (ticker === undefined) return;
178
+ clearInterval(ticker);
179
+ ticker = undefined;
180
+ stream.write("\n");
181
+ },
182
+ };
183
+ }