@skyramp/mcp 0.4.4 → 0.5.0-rc.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/build/entitlements/entitlementGate.d.ts +8 -0
  2. package/build/entitlements/entitlementGate.js +57 -0
  3. package/build/entitlements/runCodes.d.ts +18 -0
  4. package/build/entitlements/runCodes.js +16 -0
  5. package/build/index.js +32 -6
  6. package/build/playwright/PlaywrightTraceService.d.ts +17 -0
  7. package/build/playwright/PlaywrightTraceService.js +29 -1
  8. package/build/playwright/dataSafetyConfig.d.ts +74 -0
  9. package/build/playwright/dataSafetyConfig.js +109 -0
  10. package/build/playwright/essentialTools.d.ts +1 -0
  11. package/build/playwright/essentialTools.js +44 -0
  12. package/build/playwright/registerPlaywrightTools.d.ts +36 -1
  13. package/build/playwright/registerPlaywrightTools.js +87 -43
  14. package/build/playwright/safeResolveDataSafety.d.ts +24 -0
  15. package/build/playwright/safeResolveDataSafety.js +33 -0
  16. package/build/prompts/initialize-workspace/initializeWorkspacePrompt.d.ts +17 -0
  17. package/build/prompts/initialize-workspace/initializeWorkspacePrompt.js +46 -177
  18. package/build/prompts/modularization/ui-test-modularization.d.ts +1 -0
  19. package/build/prompts/modularization/ui-test-modularization.js +9 -2
  20. package/build/prompts/test-maintenance/driftAnalysisShared.js +1 -1
  21. package/build/services/FleetExecutionService.d.ts +87 -16
  22. package/build/services/FleetExecutionService.js +487 -48
  23. package/build/services/FleetRunRegistry.d.ts +49 -0
  24. package/build/services/FleetRunRegistry.js +178 -0
  25. package/build/services/containerEnv.js +8 -2
  26. package/build/services/fleetLimitConfirmation.d.ts +19 -0
  27. package/build/services/fleetLimitConfirmation.js +126 -0
  28. package/build/tool-phases.d.ts +14 -8
  29. package/build/tool-phases.js +88 -43
  30. package/build/tools/code-refactor/assertion-state.js +3 -1
  31. package/build/tools/code-refactor/codeReuseTool.js +2 -1
  32. package/build/tools/code-refactor/utils-verify-gates.js +1 -1
  33. package/build/tools/executeFleetTool.d.ts +6 -4
  34. package/build/tools/executeFleetTool.js +67 -23
  35. package/build/tools/fixErrorTool.js +2 -1
  36. package/build/tools/fleetLifecycleTools.d.ts +20 -0
  37. package/build/tools/fleetLifecycleTools.js +124 -0
  38. package/build/tools/generate-tests/generateIntegrationRestTool.d.ts +38 -0
  39. package/build/tools/generate-tests/generateIntegrationRestTool.js +7 -1
  40. package/build/tools/generate-tests/generateUILoadRestTool.d.ts +19 -1
  41. package/build/tools/generate-tests/generateUILoadRestTool.js +316 -50
  42. package/build/tools/submitReportTool.js +4 -122
  43. package/build/tools/test-management/registerTestPlanTool.js +22 -6
  44. package/build/tools/workspace/initScanWorkspaceTool.d.ts +11 -1
  45. package/build/tools/workspace/initScanWorkspaceTool.js +111 -9
  46. package/build/tools/workspace/initializeWorkspaceTool.js +40 -4
  47. package/build/types/AssertionOutcome.d.ts +3 -0
  48. package/build/types/TestbotReport.d.ts +0 -37
  49. package/build/types/TestbotReport.js +0 -22
  50. package/build/types/index.d.ts +1 -1
  51. package/build/types/index.js +1 -1
  52. package/build/utils/assertion-verify/created-add-guard.d.ts +12 -0
  53. package/build/utils/assertion-verify/created-add-guard.js +173 -0
  54. package/build/utils/assertion-verify/index.d.ts +1 -0
  55. package/build/utils/assertion-verify/index.js +1 -0
  56. package/build/utils/assertion-verify/verify.d.ts +7 -3
  57. package/build/utils/assertion-verify/verify.js +9 -3
  58. package/build/utils/featureFlags.js +6 -3
  59. package/build/utils/repositorySlug.d.ts +14 -0
  60. package/build/utils/repositorySlug.js +46 -1
  61. package/build/utils/telemetry.d.ts +2 -0
  62. package/build/utils/telemetry.js +12 -0
  63. package/build/utils/utils-verify/action-key.d.ts +59 -2
  64. package/build/utils/utils-verify/action-key.js +162 -6
  65. package/build/utils/utils-verify/action-sites.d.ts +7 -0
  66. package/build/utils/utils-verify/action-sites.js +277 -35
  67. package/build/utils/utils-verify/call-sites.d.ts +8 -2
  68. package/build/utils/utils-verify/language-spec.js +1 -1
  69. package/build/utils/utils-verify/parse.d.ts +4 -0
  70. package/build/utils/utils-verify/parse.js +2 -1
  71. package/build/utils/versions.d.ts +3 -3
  72. package/build/utils/versions.js +1 -1
  73. package/build/workspace/workspace.d.ts +63 -121
  74. package/build/workspace/workspace.js +45 -48
  75. package/build/workspace/workspaceVerifier.d.ts +22 -0
  76. package/build/workspace/workspaceVerifier.js +65 -0
  77. package/node_modules/playwright/lib/dom-analyzer/blueprint.js +7 -6
  78. package/node_modules/playwright/lib/dom-analyzer/dynamicId.js +5 -4
  79. package/node_modules/playwright/lib/dom-analyzer/dynamicId.test.js +12 -0
  80. package/node_modules/playwright/lib/mcp/skyramp/dataSafetyGuard.js +822 -0
  81. package/node_modules/playwright/lib/mcp/skyramp/dataSafetySession.js +372 -0
  82. package/node_modules/playwright/lib/mcp/skyramp/exportTool.js +101 -8
  83. package/node_modules/playwright/lib/mcp/skyramp/finishTestSetupTool.js +45 -0
  84. package/node_modules/playwright/lib/mcp/skyramp/index.js +16 -0
  85. package/node_modules/playwright/lib/mcp/skyramp/recordingPhase.js +65 -0
  86. package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +1 -0
  87. package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +598 -127
  88. package/node_modules/playwright/lib/mcp/test/skyRampExport.js +85 -31
  89. package/node_modules/playwright/lib/mcp/test/skyRampExport.test.js +132 -5
  90. package/node_modules/playwright/node_modules/playwright-core/lib/cli/program.js +2 -0
  91. package/node_modules/playwright/node_modules/playwright-core/lib/client/browserContext.js +4 -2
  92. package/node_modules/playwright/node_modules/playwright-core/lib/generated/injectedScriptSource.js +1 -1
  93. package/node_modules/playwright/node_modules/playwright-core/lib/generated/pollingRecorderSource.js +1 -1
  94. package/node_modules/playwright/node_modules/playwright-core/lib/protocol/validator.js +2 -1
  95. package/node_modules/playwright/node_modules/playwright-core/lib/server/browserContext.js +17 -1
  96. package/node_modules/playwright/node_modules/playwright-core/lib/server/codegen/skyramp/jsonlReader.js +1 -0
  97. package/node_modules/playwright/node_modules/playwright-core/lib/server/har/harRecorder.js +1 -0
  98. package/node_modules/playwright/node_modules/playwright-core/lib/server/har/harTracer.js +1 -1
  99. package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder/recorderRunner.js +2 -1
  100. package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder/recorderSignalProcessor.js +2 -1
  101. package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder/skyramp/clickExpandedState.js +75 -0
  102. package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder.js +4 -2
  103. package/node_modules/playwright/node_modules/playwright-core/lib/utils/isomorphic/volatileDate.js +85 -0
  104. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-Bou8kKzE.js → codeMirrorModule-BuhAdckO.js} +1 -1
  105. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-COAHN430.js +809 -0
  106. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.0MJW0uG7.js → index.7AU8h0PZ.js} +1 -1
  107. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
  108. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{uiMode.GLqsx5cI.js → uiMode.Dl7smBJn.js} +1 -1
  109. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
  110. package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
  111. package/node_modules/playwright/node_modules/playwright-core/src/cli/program.ts +5 -0
  112. package/node_modules/playwright/node_modules/playwright-core/src/client/browserContext.ts +3 -1
  113. package/node_modules/playwright/node_modules/playwright-core/src/client/types.ts +2 -0
  114. package/node_modules/playwright/node_modules/playwright-core/src/generated/injectedScriptSource.ts +1 -1
  115. package/node_modules/playwright/node_modules/playwright-core/src/generated/pollingRecorderSource.ts +1 -1
  116. package/node_modules/playwright/node_modules/playwright-core/src/protocol/validator.ts +1 -0
  117. package/node_modules/playwright/node_modules/playwright-core/src/server/browserContext.ts +25 -1
  118. package/node_modules/playwright/node_modules/playwright-core/src/server/codegen/skyramp/jsonlReader.ts +2 -0
  119. package/node_modules/playwright/node_modules/playwright-core/src/server/har/harRecorder.ts +1 -0
  120. package/node_modules/playwright/node_modules/playwright-core/src/server/har/harTracer.ts +3 -1
  121. package/node_modules/playwright/node_modules/playwright-core/src/server/recorder/recorderRunner.ts +3 -1
  122. package/node_modules/playwright/node_modules/playwright-core/src/server/recorder/recorderSignalProcessor.ts +1 -0
  123. package/node_modules/playwright/node_modules/playwright-core/src/server/recorder/skyramp/clickExpandedState.ts +82 -0
  124. package/node_modules/playwright/node_modules/playwright-core/src/server/recorder.ts +2 -0
  125. package/node_modules/playwright/node_modules/playwright-core/src/utils/isomorphic/volatileDate.ts +137 -1
  126. package/node_modules/playwright/package.json +1 -1
  127. package/package.json +4 -4
  128. package/plugin/prompts/plan-tests.md +2 -2
  129. package/plugin/prompts/testbot-task1.md +1 -1
  130. package/plugin/skills/enhance-assertions/reference/integration.md +1 -0
  131. package/plugin/skills/setup-testbot-target-sut/SKILL.md +60 -0
  132. package/plugin/skills/setup-testbot-target-sut/reference/ci-workflow.md +26 -0
  133. package/plugin/skills/setup-testbot-target-sut/reference/deployed-environment.md +105 -0
  134. package/plugin/skills/setup-testbot-target-sut/reference/existing-tests.md +14 -0
  135. package/plugin/skills/setup-testbot-target-sut/reference/troubleshooting.md +20 -0
  136. package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.d.ts +0 -2
  137. package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.js +0 -178
  138. package/build/prompts/sut-setup/modes/dockerComposePrompt.d.ts +0 -2
  139. package/build/prompts/sut-setup/modes/dockerComposePrompt.js +0 -311
  140. package/build/prompts/sut-setup/shared.d.ts +0 -62
  141. package/build/prompts/sut-setup/shared.js +0 -196
  142. package/build/resources/sutSetupResource.d.ts +0 -2
  143. package/build/resources/sutSetupResource.js +0 -63
  144. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-DTp2-8Si.js +0 -809
@@ -1,9 +1,12 @@
1
1
  import { SkyrampClient } from "@skyramp/skyramp";
2
2
  import { z } from "zod";
3
- import { existsSync, readFileSync } from "fs";
3
+ import semver from "semver";
4
+ import { appendFileSync, copyFileSync, existsSync, readFileSync } from "fs";
5
+ import * as path from "path";
4
6
  import { ProgrammingLanguage } from "../../types/TestTypes.js";
5
7
  import { toolError, validatePath } from "../../utils/utils.js";
6
8
  import { logger } from "../../utils/logger.js";
9
+ import { SKYRAMP_SDK_VERSION } from "../../utils/skyrampSdkVersion.js";
7
10
  import { AnalyticsService } from "../../services/AnalyticsService.js";
8
11
  import { generatedFilesFrom } from "../../services/TestGenerationService.js";
9
12
  import { getTestsRepoDir } from "../../utils/AnalysisStateManager.js";
@@ -12,6 +15,7 @@ import { resolveOutputDir, stageGeneratedPaths, } from "../../utils/gitStaging.j
12
15
  import { normalizeLanguageParams } from "../../utils/normalizeParams.js";
13
16
  import { getEntryPoint } from "../../utils/telemetry.js";
14
17
  import { checkTraceReuse, recordTraceOutput } from "./trace-reuse-guard.js";
18
+ import { limitConfirmationToken, readFleetFileLimits, } from "../../services/fleetLimitConfirmation.js";
15
19
  const TOOL_NAME = "skyramp_ui_load_test_generation";
16
20
  /**
17
21
  * Language and framework in the spelling the generator takes. Done in the SCHEMA rather than
@@ -83,8 +87,9 @@ export const uiLoadTestSchema = {
83
87
  .string()
84
88
  .default("")
85
89
  .describe("Remote Docker Engine where the worker for this host is deployed, e.g. " +
86
- '"tcp://10.0.1.11:2376". Empty means the local/default Docker context, which ' +
87
- "skyramp_execute_fleet cannot run: run such a fleet on the host instead."),
90
+ '"tcp://10.0.1.11:2376". Empty means the local Docker daemon: the worker is ' +
91
+ "deployed on this machine and no certificates are needed, which is how to try a " +
92
+ "fleet before pointing it at regions."),
88
93
  tls: z
89
94
  .string()
90
95
  .default("")
@@ -122,6 +127,14 @@ export const uiLoadTestSchema = {
122
127
  .default("")
123
128
  .describe("Tag for workerImage. Empty uses the library's default. skyramp_execute_fleet " +
124
129
  "refuses a fleet that sets it: run such a fleet on the host."),
130
+ browserSessions: z
131
+ .number()
132
+ .int()
133
+ .min(0)
134
+ .default(0)
135
+ .describe("Browser sessions this host runs, fixed. 0 (the default) sizes it from the host's " +
136
+ "CPU cores (see coresPerBrowser). Java, python, typescript and javascript. Set it when the user names a browser " +
137
+ 'count per host ("4 browsers on each host").'),
125
138
  }))
126
139
  .min(1)
127
140
  // The fleet runner's own rules (pkg/fleet/runner.go validate), applied here so a host
@@ -161,17 +174,6 @@ export const uiLoadTestSchema = {
161
174
  .describe("The load-generating hosts to fan the journey across, one entry per VM or region. " +
162
175
  "Each becomes an inline Host(...) in the generated fleet manager. Ask the user which " +
163
176
  "regions they want to drive load from; at least one is required."),
164
- browserCoverage: z
165
- .number()
166
- // Strictly inside 0..1, not inclusive: ui-load always emits BOTH halves, and the fleet
167
- // refuses a run carrying both unless the virtual users actually divide between them. 0
168
- // gives each half the full count; 1 leaves the protocol half none.
169
- .gt(0)
170
- .lt(1)
171
- .default(0.1)
172
- .describe("Fraction of the run executed in REAL browsers, strictly between 0 and 1; the rest " +
173
- "runs as the cheaper protocol load. Default 0.1. Raise toward 1 for browser " +
174
- "fidelity, lower for mostly protocol load. Do NOT assume a non-default value."),
175
177
  loadDuration: z
176
178
  .number()
177
179
  .int()
@@ -185,9 +187,11 @@ export const uiLoadTestSchema = {
185
187
  .positive()
186
188
  // 2, not 1: both halves always ship, so the smallest honest run is one user in each.
187
189
  .default(2)
188
- .describe("Total virtual users (default 2), split between the browser and protocol halves by " +
189
- "browserCoverage and then across the hosts. Must be large enough to give every " +
190
- "host at least one user in each half; a smaller count is refused with the minimum."),
190
+ .describe('Total user sessions (default 2), the number the user asks for ("100 users"). The ' +
191
+ "fleet splits them by host capacity: each host runs as many real-browser sessions " +
192
+ "as it can carry and protocol users for the rest of its share, so the run provisions " +
193
+ "exactly this many. Capacity follows pacing: one browser per CPU core when " +
194
+ "loadTargetRPS is set, a quarter of the cores when it is not. At least one per host."),
191
195
  loadCount: z
192
196
  .number()
193
197
  .int()
@@ -199,7 +203,18 @@ export const uiLoadTestSchema = {
199
203
  .int()
200
204
  .nonnegative()
201
205
  .default(0)
202
- .describe("Target requests per second. 0 means as fast as the virtual users go."),
206
+ .describe("Pace for the protocol users, in scenario iterations per second for the whole run " +
207
+ "(one iteration replays every recorded request once). 0 means as fast as the " +
208
+ "application answers, which takes most of each host's CPU and leaves the browsers " +
209
+ "one per four cores. Set it when the user asks for a rate or wants browser timings " +
210
+ "that describe the application rather than the test machine."),
211
+ coresPerBrowser: z
212
+ .number()
213
+ .positive()
214
+ .optional()
215
+ .describe("CPU cores given to each browser session when the fleet sizes a host's browser " +
216
+ "capacity, e.g. 1 for one browser per core or 0.5 for two. Omit it and capacity " +
217
+ "follows pacing (1 when loadTargetRPS is set, 4 when not). Java, python, typescript and javascript."),
203
218
  loadRampupDuration: z
204
219
  .number()
205
220
  .int()
@@ -212,6 +227,44 @@ export const uiLoadTestSchema = {
212
227
  .nonnegative()
213
228
  .default(0)
214
229
  .describe("Seconds between ramp-up steps."),
230
+ limits: z
231
+ .array(z.string().min(1))
232
+ .optional()
233
+ .describe("Performance pass conditions, validated by the native KPI parser. Read " +
234
+ "skyramp_kpi_catalog first for native names, units, scopes and site defaults. " +
235
+ "Syntax: '<kpi>[{scope}] [<agg>] <op> <value> [fail|warn] [for <duration>] [stop]'; " +
236
+ "'for' holds a breach for that long during the run before it trips, 'stop' ends the " +
237
+ "run when a fail limit trips. Examples: 'backend_latency p95 < 1500ms fail', " +
238
+ "'backend_latency p95 < 1200ms warn for 60s', 'error_rate < 5% fail for 30s stop', " +
239
+ "'lcp{region:eu-west} p90 < 2500ms fail'. " +
240
+ "KPIs: 'the slowest 5% of API calls', 'p95', '95th percentile' -> backend_latency p95; " +
241
+ "'API calls', 'requests', 'the backend' (latency) -> backend_latency; " +
242
+ "'pages render/load in', 'page load' -> lcp p90; 'responsiveness', 'interaction' -> " +
243
+ "inp p90; 'layout shift', 'jumpy' -> cls p90; 'errors', 'error rate', 'failed " +
244
+ "requests' -> error_rate; 'journeys don't finish', 'complete', 'success' -> " +
245
+ "journey_success (a minimum rate, >=). ttfb is not a catalogue entry: ASK, offering " +
246
+ "backend_latency or lcp, rather than substitute silently. " +
247
+ "PASS condition: 'go over/exceed/more than X' -> <= X (fail is strictly > X); " +
248
+ "'within X' -> <= X (inclusive); 'under/less than X' -> < X (exclusive); " +
249
+ "'at least X%' -> >= X%. Units: '1.5 seconds' -> 1500ms, 'two seconds' -> 2000ms, " +
250
+ "'2%' -> 2% (0.02), '99%' -> 99% (0.99). " +
251
+ "Severity: 'fail', 'fail the build', 'the run must' -> fail (the default); 'warn', " +
252
+ "'tell me', 'flag', 'let me know', 'heading there' -> warn ('flag' always means warn). " +
253
+ "Holds: 'for more than a minute' -> for 60s; 'for half a minute' -> for 30s; a bare " +
254
+ "'warn if X hits Y' uses a 60s warning hold (echo it explicitly); 'a single bad spike " +
255
+ "is fine' needs a for: ASK for the duration if none is given. " +
256
+ "Stop: 'end the run when'/'stop when' -> append stop, with a 30s hold when none is " +
257
+ "given (echo it). Only fail limits can stop: if the user asks to warn AND stop, ASK " +
258
+ "whether they mean fail+stop instead. 'don't stop the run' never gets stop. " +
259
+ "Scopes: 'in the UK' -> {region:uk}; a host by name -> {host:name}; the region or " +
260
+ "host must exist in the run, ASK if it does not. " +
261
+ "ASK, do not guess, when a unit is ambiguous or limits on one KPI/scope contradict " +
262
+ "(e.g. < 1000ms and > 2000ms). " +
263
+ "Translate to pass conditions; do not invent units, durations or unknown scopes. " +
264
+ "Omit when the user states none: no default limits are added — site defaults are " +
265
+ "'suggested, NOT applied', offer them but add only if asked. Echo the normalized " +
266
+ "limits this tool returns (scope, severity, hold) and WAIT for user confirmation " +
267
+ "before passing confirmationToken to skyramp_start_fleet or skyramp_execute_fleet."),
215
268
  thresholds: z
216
269
  .string()
217
270
  .default("")
@@ -241,6 +294,13 @@ export const uiLoadTestSchema = {
241
294
  .string()
242
295
  .min(1)
243
296
  .describe("Absolute path to the directory the three files are written to."),
297
+ credentialsFile: z
298
+ .string()
299
+ .optional()
300
+ .describe("Absolute path to the credential pool (CSV with a `token` column, or `username,password`) " +
301
+ "the fleet hands its virtual users. Copied next to the generated files as " +
302
+ "credentials.csv, where the generated fleet reads it. Default: a credentials.csv " +
303
+ "beside playwrightStoragePath."),
244
304
  allowTraceReuse: z
245
305
  .boolean()
246
306
  .default(false)
@@ -255,29 +315,95 @@ export const uiLoadTestSchema = {
255
315
  .describe("Overwrite existing files with the same names (default true). Set false to keep " +
256
316
  "files that are already there."),
257
317
  };
318
+ /**
319
+ * The first @skyramp/skyramp release whose native fleet runner sizes `browserCoverage: 0`
320
+ * by host capacity and reads a `{hosts, coresPerBrowser}` envelope. Both changed meaning
321
+ * from the released sizing model (0 used to mean "give each half the full count", and
322
+ * hosts was always an array), entirely on the native side: the JS wrapper forwards either
323
+ * shape unchanged, so unlike playwrightStorage there is no source text to detect the
324
+ * capability from, only the version this code was written against.
325
+ */
326
+ const MIN_SIZING_SDK_VERSION = "1.3.50";
258
327
  /** The line the generator prints for each file it writes. */
259
328
  const GENERATED_FILES_LINE = /Successfully generated\/updated tests? files?:/i;
329
+ /** The generated fleet manager, named as skyramp_execute_fleet discovers it. */
330
+ const FLEET_MANAGER_FILE = /(FleetTest\.java|_fleet\.(py|ts|js))$/;
260
331
  /**
261
- * The smallest virtual-user count, at or above `threads`, that gives every host at least
262
- * one user in each half -- or null when `threads` already does.
263
- *
264
- * Mirrors the fleet (pkg/fleet/runner.go): browserShare splits the total between the halves,
265
- * rounding the browser slice to nearest and flooring it at 1, and splitShare then divides
266
- * each half across the hosts, which this tool always weights equally. The worker runs a
267
- * share of zero as ONE session, so a count too small to divide does not fail: it quietly
268
- * runs more users than the generated file names.
332
+ * Legacy threshold keys in the limit form the fleet core maps them to (D7): inclusive
333
+ * bounds, evaluated per region and overall, CPU per host. For the echo only; the
334
+ * thresholds themselves still reach the generator unchanged.
269
335
  */
270
- function minimumThreads(threads, hosts, coverage) {
271
- const divides = (total) => {
272
- const browser = Math.min(Math.max(Math.round(total * coverage), 1), total);
273
- return browser >= hosts && total - browser >= hosts;
274
- };
275
- if (divides(threads))
276
- return null;
277
- let total = threads + 1;
278
- while (!divides(total))
279
- total++;
280
- return total;
336
+ export function legacyThresholdLimits(thresholds) {
337
+ const lines = [];
338
+ for (const pair of thresholds.split(",")) {
339
+ const [key, value] = pair.split("=").map((p) => p?.trim() ?? "");
340
+ if (!key)
341
+ continue;
342
+ const line = {
343
+ lcp_ms: `lcp p90 <= ${value}ms fail`,
344
+ inp_ms: `inp p90 <= ${value}ms fail`,
345
+ cls: `cls p90 <= ${value} fail`,
346
+ journey_success_rate: `journey_success >= ${value} fail`,
347
+ generator_cpu_pct: `generator_cpu p50 <= ${value}% fail`,
348
+ }[key];
349
+ if (line)
350
+ lines.push(line);
351
+ else if (key === "allowUnmeasured")
352
+ lines.push(`allowUnmeasured=${value} (an unmeasured threshold passes)`);
353
+ else
354
+ lines.push(`${key}=${value} (not a known threshold key)`);
355
+ }
356
+ return lines;
357
+ }
358
+ /**
359
+ * What the generation tells the agent to echo before any run. Explicit limits come with
360
+ * the confirmation token the runners require. Legacy thresholds are shown as the limits
361
+ * they map to. With neither the fleet gates on nothing, and the site defaults are only
362
+ * offered as suggestions (D5 does not apply them).
363
+ */
364
+ function limitEcho(client, params, fileLimits, token) {
365
+ const parts = [];
366
+ if (fileLimits.length > 0) {
367
+ parts.push("Limits set (normalized):\n" +
368
+ fileLimits.map((l) => `- ${l}`).join("\n") +
369
+ `\nconfirmationToken: ${token}\n` +
370
+ "Echo these pass conditions to the user and WAIT for their confirmation. Only after " +
371
+ "they confirm, pass confirmationToken to skyramp_start_fleet (then poll " +
372
+ "skyramp_fleet_status and collect skyramp_fleet_result) or skyramp_execute_fleet; " +
373
+ "both refuse this fleet without it.");
374
+ }
375
+ if (params.thresholds) {
376
+ const mapped = legacyThresholdLimits(params.thresholds).map((line) => {
377
+ if (!client.normalizeFleetLimits || line.includes("("))
378
+ return line;
379
+ try {
380
+ return client.normalizeFleetLimits([line], params.hosts)[0] ?? line;
381
+ }
382
+ catch {
383
+ return line;
384
+ }
385
+ });
386
+ parts.push("Legacy thresholds set, enforced with their original behavior (per region and " +
387
+ "overall; generator CPU per host):\n" +
388
+ mapped.map((l) => `- ${l}`).join("\n") +
389
+ "\nEcho these to the user before running the fleet.");
390
+ }
391
+ if (parts.length === 0) {
392
+ let suggested = [];
393
+ try {
394
+ suggested = client.getFleetLimitCatalogue?.().defaults ?? [];
395
+ }
396
+ catch {
397
+ // An SDK without the catalogue has no suggestions to offer.
398
+ }
399
+ parts.push("No limits set: this fleet gates on no performance limit." +
400
+ (suggested.length > 0
401
+ ? "\nSuggested site defaults (suggested, NOT applied):\n" +
402
+ suggested.map((l) => `- ${l}`).join("\n") +
403
+ "\nOffer them to the user; regenerate with them as limits only if the user asks."
404
+ : ""));
405
+ }
406
+ return parts.join("\n\n");
281
407
  }
282
408
  /**
283
409
  * Every file the generator reports writing. ui-load prints one "Successfully generated"
@@ -357,6 +483,9 @@ export async function generateUILoadTest(params) {
357
483
  const outError = validatePath(params.outputDir, "outputDir");
358
484
  if (outError)
359
485
  return outError;
486
+ const credentialsError = validatePath(params.credentialsFile ?? "", "credentialsFile");
487
+ if (credentialsError)
488
+ return credentialsError;
360
489
  // The filesystem guards every other generator gets from TestGenerationService, which
361
490
  // this tool does not pass through because ui-load has its own FFI export.
362
491
  // In cross-repo mode the files belong in the tests repo clone, not the source repo.
@@ -366,17 +495,23 @@ export async function generateUILoadTest(params) {
366
495
  const badTarget = generationTargets(outputDir, params.output || undefined).find(isInsideExecutorWorkDir);
367
496
  if (badTarget)
368
497
  return toolError(executorWorkDirRefusal(badTarget));
369
- const minimum = minimumThreads(params.loadNumThreads, params.hosts.length, params.browserCoverage);
370
- if (minimum !== null) {
498
+ // The fleet refuses this too; refusing here says so before any file is written.
499
+ if (params.loadNumThreads < params.hosts.length) {
371
500
  return toolError(`Error: loadNumThreads ${params.loadNumThreads} cannot give each of the ` +
372
- `${params.hosts.length} host(s) a virtual user in both halves at browserCoverage ` +
373
- `${params.browserCoverage}. Use at least ${minimum}, or change browserCoverage.`);
501
+ `${params.hosts.length} host(s) a session. Use at least ${params.hosts.length}.`);
374
502
  }
375
503
  const client = new SkyrampClient();
376
504
  if (typeof client.generateUILoad !== "function") {
377
505
  return toolError("Error: the installed @skyramp/skyramp SDK does not expose generateUILoad. " +
378
506
  "Upgrade to an SDK that includes the ui-load generator.");
379
507
  }
508
+ // Sent on every call, not only when coresPerBrowser is set: an SDK below this floor reads
509
+ // browserCoverage 0 and an array-only hosts shape under the OLD sizing model, which
510
+ // silently doubles the provisioned users rather than failing loudly.
511
+ if (semver.lt(SKYRAMP_SDK_VERSION, MIN_SIZING_SDK_VERSION)) {
512
+ return toolError(`Error: the installed @skyramp/skyramp SDK (${SKYRAMP_SDK_VERSION}) predates host-capacity ` +
513
+ `sizing for ui-load. Upgrade to ${MIN_SIZING_SDK_VERSION} or later, which this generator requires.`);
514
+ }
380
515
  // The released SDK has generateUILoad but never reads playwrightStorage, so the session
381
516
  // would be dropped and every virtual user would run unauthenticated, with a success
382
517
  // message on top. The function's own source is the only signal the SDK offers: it has no
@@ -392,6 +527,22 @@ export async function generateUILoadTest(params) {
392
527
  hosts: params.hosts.length,
393
528
  outputDir,
394
529
  });
530
+ // Without limits the generation is the one older SDKs know: the limit functions are
531
+ // never needed, and no default limits are added (D5 defaults are only suggested).
532
+ let normalizedLimits = [];
533
+ if (params.limits && params.limits.length > 0) {
534
+ if (typeof client.normalizeFleetLimits !== "function") {
535
+ return toolError("Error: the installed @skyramp/skyramp SDK is too old for limits: it cannot " +
536
+ "validate them natively. Upgrade the SDK, or generate without limits.");
537
+ }
538
+ try {
539
+ normalizedLimits = client.normalizeFleetLimits(params.limits, params.hosts);
540
+ }
541
+ catch (error) {
542
+ const message = error instanceof Error ? error.message : String(error);
543
+ return toolError(`Error: ui-load generation failed: ${message}`);
544
+ }
545
+ }
395
546
  let result;
396
547
  try {
397
548
  result = await client.generateUILoad({
@@ -410,9 +561,12 @@ export async function generateUILoadTest(params) {
410
561
  loadRampupDuration: params.loadRampupDuration,
411
562
  loadRampupInterval: params.loadRampupInterval,
412
563
  loadTargetRPS: params.loadTargetRPS,
413
- browserCoverage: params.browserCoverage,
564
+ browserCoverage: 0,
414
565
  thresholds: params.thresholds,
415
- hosts: params.hosts,
566
+ ...(normalizedLimits.length > 0 ? { limits: normalizedLimits } : {}),
567
+ hosts: params.coresPerBrowser !== undefined
568
+ ? { hosts: params.hosts, coresPerBrowser: params.coresPerBrowser }
569
+ : params.hosts,
416
570
  deployWorker: params.deployWorker,
417
571
  entrypoint: getEntryPoint(),
418
572
  });
@@ -439,25 +593,137 @@ export async function generateUILoadTest(params) {
439
593
  "@skyramp/skyramp SDK ignored it, so the fleet cannot run as generated. Upgrade to " +
440
594
  "an SDK that supports per-host hostPort and tls.");
441
595
  }
442
- // Stage so testbot includes the generated files in its output commit.
443
- await stageGeneratedPaths(outputDir);
596
+ // The limits the fleet will actually run, read back from what was written: the token
597
+ // the runners require is computed from the same file.
598
+ const fleetFile = files.find((f) => FLEET_MANAGER_FILE.test(f));
599
+ let fileLimits = [];
600
+ let token;
601
+ if (normalizedLimits.length > 0) {
602
+ const fleetPath = fleetFile ? path.resolve(outputDir, fleetFile) : "";
603
+ try {
604
+ fileLimits =
605
+ fleetPath && existsSync(fleetPath)
606
+ ? readFleetFileLimits(fleetPath)
607
+ : [];
608
+ }
609
+ catch (error) {
610
+ return toolError(`Error: ${error instanceof Error ? error.message : String(error)}`);
611
+ }
612
+ // The generator writes the normalized lines verbatim, so any difference means the SDK
613
+ // dropped or changed a requested limit, and the token would confirm a different run.
614
+ const missing = normalizedLimits.filter((l) => !fileLimits.includes(l));
615
+ const unexpected = fileLimits.filter((l) => !normalizedLimits.includes(l));
616
+ if (missing.length > 0 || unexpected.length > 0) {
617
+ return toolError("Error: the generated fleet file does not carry exactly the requested limits." +
618
+ (missing.length > 0 ? ` Missing: ${missing.join("; ")}.` : "") +
619
+ (unexpected.length > 0
620
+ ? ` Unexpected: ${unexpected.join("; ")}.`
621
+ : "") +
622
+ " The installed @skyramp/skyramp SDK ignored or changed them; upgrade it to " +
623
+ "generate limits.");
624
+ }
625
+ token = limitConfirmationToken(fleetPath, fileLimits);
626
+ }
627
+ const poolNote = placeCredentialPool(files, outputDir, params);
628
+ // Stage an explicit allowlist, not outputDir itself: `git add <dir>` restages any path
629
+ // under it that's already tracked regardless of .gitignore, so bulk-staging could
630
+ // recommit the credential pool if this outputDir once had it tracked and deleted.
631
+ const gitignorePath = path.join(outputDir, ".gitignore");
632
+ const pathsToStage = existsSync(gitignorePath)
633
+ ? [...files, gitignorePath]
634
+ : files;
635
+ for (const generatedPath of pathsToStage) {
636
+ await stageGeneratedPaths(generatedPath);
637
+ }
444
638
  return {
445
639
  content: [
446
640
  { type: "text", text: result },
641
+ { type: "text", text: limitEcho(client, params, fileLimits, token) },
447
642
  {
448
643
  type: "text",
449
644
  text: `Generated three coordinated files in ${outputDir}:\n` +
450
645
  `- the UI test, which replays the recorded journey in a real browser\n` +
451
646
  `- the load file, whose function RETURNS the protocol scenario (static assets filtered out)\n` +
452
647
  `- the fleet manager, which wires the hosts, the UI test and the scenario into one run\n\n` +
453
- `The fleet manager is the entry point. For python/typescript/javascript run it as a ` +
454
- `script; for java run it as a JUnit test. It builds the scenario, fans the journey ` +
455
- `across the hosts at browserCoverage ${params.browserCoverage}, and asserts the ` +
456
- `thresholds.`,
648
+ `The fleet manager is the entry point; run it with skyramp_execute_fleet (java, ` +
649
+ `python, typescript, javascript), which runs it in the executor container. It builds the scenario, runs ` +
650
+ `${params.loadNumThreads} sessions across the hosts for ${params.loadDuration}s -- as ` +
651
+ `many real browsers as each host can carry, protocol users for the rest -- and ` +
652
+ `asserts the thresholds.` +
653
+ (poolNote ? `\n\n${poolNote}` : ""),
457
654
  },
458
655
  ],
459
656
  };
460
657
  }
658
+ /** The file the generated fleet reads its identities from (SKYRAMP_CREDENTIALS default). */
659
+ const CREDENTIALS_FILENAME = "credentials.csv";
660
+ /**
661
+ * Copy the credential pool (credentialsFile, else credentials.csv beside the recorded
662
+ * session) to credentials.csv beside the generated files, where the fleet reads it; an
663
+ * existing one is left alone. Returns a line for the tool's result, or "" when not needed.
664
+ */
665
+ export function placeCredentialPool(files, outputDir, params) {
666
+ const note = copyCredentialPool(files, outputDir, params);
667
+ return note && note + ignoreCredentialPool(outputDir);
668
+ }
669
+ function copyCredentialPool(files, outputDir, params) {
670
+ const fleetFile = files.find((f) => FLEET_MANAGER_FILE.test(f));
671
+ if (!fleetFile)
672
+ return "";
673
+ let fleetSource = "";
674
+ try {
675
+ fleetSource = readFileSync(path.resolve(outputDir, fleetFile), "utf8");
676
+ }
677
+ catch {
678
+ return "";
679
+ }
680
+ if (!fleetSource.includes(CREDENTIALS_FILENAME))
681
+ return "";
682
+ const target = path.join(outputDir, CREDENTIALS_FILENAME);
683
+ if (existsSync(target)) {
684
+ return `Credential pool: the fleet reads ${target}, which is already there.`;
685
+ }
686
+ const candidates = [
687
+ params.credentialsFile,
688
+ params.playwrightStoragePath
689
+ ? path.join(path.dirname(params.playwrightStoragePath), CREDENTIALS_FILENAME)
690
+ : undefined,
691
+ ].filter((p) => !!p);
692
+ const source = candidates.find((p) => existsSync(p));
693
+ if (!source) {
694
+ return (`Credential pool: the fleet reads ${target}, and none was found ` +
695
+ `(looked for ${candidates.join(", ") || "credentialsFile"}). Put the pool there, or ` +
696
+ "pass credentialsFile, before running the fleet.");
697
+ }
698
+ try {
699
+ copyFileSync(source, target);
700
+ }
701
+ catch (error) {
702
+ return (`Credential pool: could not copy ${source} to ${target} ` +
703
+ `(${error instanceof Error ? error.message : String(error)}). Copy it there, or ` +
704
+ "pass credentialsFile, before running the fleet.");
705
+ }
706
+ return `Credential pool: copied ${source} to ${target}, where the fleet reads it.`;
707
+ }
708
+ /** Lists the pool in <outputDir>/.gitignore so testbot's staging never commits it. */
709
+ function ignoreCredentialPool(outputDir) {
710
+ const ignoreFile = path.join(outputDir, ".gitignore");
711
+ try {
712
+ const existing = existsSync(ignoreFile)
713
+ ? readFileSync(ignoreFile, "utf8")
714
+ : "";
715
+ if (existing.split(/\r?\n/).some((l) => l.trim() === CREDENTIALS_FILENAME)) {
716
+ return "";
717
+ }
718
+ const separator = existing === "" || existing.endsWith("\n") ? "" : "\n";
719
+ appendFileSync(ignoreFile, `${separator}${CREDENTIALS_FILENAME}\n`);
720
+ return "";
721
+ }
722
+ catch (error) {
723
+ return (` Warning: could not add ${CREDENTIALS_FILENAME} to ${ignoreFile} ` +
724
+ `(${error instanceof Error ? error.message : String(error)}); keep the pool out of git.`);
725
+ }
726
+ }
461
727
  /**
462
728
  * The tool's entry point: the trace-reuse guard and the usage event around the generation,
463
729
  * as skyramp_ui_test_generation has them.
@@ -494,7 +760,7 @@ Emits THREE coordinated files:
494
760
 
495
761
  **Inputs you must gather from the user, not guess:**
496
762
  - \`hosts\` — which VMs or regions should generate load. At least one is required; each becomes an inline Host(...) in the fleet manager. A host on a remote Docker Engine (\`dockerHost\`) also needs \`tls\` — the directory holding its client certificates — because that connection is mTLS.
497
- - \`browserCoverage\` — how much of the run is real browsers. Strictly between 0 and 1. Do NOT assume a non-default value.
763
+ - \`loadNumThreads\` — how many user sessions to run ("100 users"). The fleet splits them by host capacity: each host runs as many real-browser sessions as it can carry and protocol users for the rest, and fails the run when it cannot provision them. Capacity defaults to one browser per CPU core when \`loadTargetRPS\` is set, a quarter of the cores otherwise; set \`coresPerBrowser\` to change that ratio, or a host's \`browserSessions\` to fix its browser count directly ("4 browsers on each host").
498
764
 
499
765
  **Exclude third-party hosts.** A trace from a real site carries CDN, analytics and ad traffic. \`exclude\` matches the EXACT host, so enumerate every one — excluding "example.com" does not drop "cdn.example.com". Without this the protocol half load-tests other people's infrastructure.
500
766