@jevitate/cli 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/dist/bin.js +48283 -47589
  2. package/dist/build-info.generated.js +2 -2
  3. package/dist/explore-cli/adversarial-strategy.d.ts +3 -0
  4. package/dist/explore-cli/adversarial-strategy.js +99 -0
  5. package/dist/explore-cli/adversarial-strategy.js.map +1 -0
  6. package/dist/explore-cli/coverage-strategy.d.ts +3 -0
  7. package/dist/explore-cli/coverage-strategy.js +95 -0
  8. package/dist/explore-cli/coverage-strategy.js.map +1 -0
  9. package/dist/explore-cli/feature-strategy.d.ts +3 -0
  10. package/dist/explore-cli/feature-strategy.js +59 -0
  11. package/dist/explore-cli/feature-strategy.js.map +1 -0
  12. package/dist/explore-cli/goal-strategy.d.ts +3 -0
  13. package/dist/explore-cli/goal-strategy.js +184 -0
  14. package/dist/explore-cli/goal-strategy.js.map +1 -0
  15. package/dist/explore-cli/options.d.ts +3 -0
  16. package/dist/explore-cli/options.js +155 -0
  17. package/dist/explore-cli/options.js.map +1 -0
  18. package/dist/explore-cli/run.d.ts +7 -0
  19. package/dist/explore-cli/run.js +682 -0
  20. package/dist/explore-cli/run.js.map +1 -0
  21. package/dist/explore-cli/shared.d.ts +158 -0
  22. package/dist/explore-cli/shared.js +20 -0
  23. package/dist/explore-cli/shared.js.map +1 -0
  24. package/dist/explore-cli/usability-strategy.d.ts +3 -0
  25. package/dist/explore-cli/usability-strategy.js +136 -0
  26. package/dist/explore-cli/usability-strategy.js.map +1 -0
  27. package/dist/explore-cli.d.ts +2 -2
  28. package/dist/explore-cli.js +2 -1347
  29. package/dist/explore-cli.js.map +1 -1
  30. package/dist/journey-anchor-api.d.ts +10 -9
  31. package/dist/journey-anchor-api.js +12 -22
  32. package/dist/journey-anchor-api.js.map +1 -1
  33. package/dist/journey-anchor-cli.d.ts +1 -1
  34. package/dist/journey-anchor-cli.js +10 -6
  35. package/dist/journey-anchor-cli.js.map +1 -1
  36. package/dist/journey-cli/anchors.d.ts +4 -0
  37. package/dist/journey-cli/anchors.js +48 -0
  38. package/dist/journey-cli/anchors.js.map +1 -0
  39. package/dist/journey-cli/find.d.ts +4 -0
  40. package/dist/journey-cli/find.js +44 -0
  41. package/dist/journey-cli/find.js.map +1 -0
  42. package/dist/journey-cli/lint.d.ts +4 -0
  43. package/dist/journey-cli/lint.js +76 -0
  44. package/dist/journey-cli/lint.js.map +1 -0
  45. package/dist/journey-cli/list.d.ts +4 -0
  46. package/dist/journey-cli/list.js +48 -0
  47. package/dist/journey-cli/list.js.map +1 -0
  48. package/dist/journey-cli/migrate.d.ts +4 -0
  49. package/dist/journey-cli/migrate.js +101 -0
  50. package/dist/journey-cli/migrate.js.map +1 -0
  51. package/dist/journey-cli/promote.d.ts +4 -0
  52. package/dist/journey-cli/promote.js +181 -0
  53. package/dist/journey-cli/promote.js.map +1 -0
  54. package/dist/journey-cli/publish.d.ts +4 -0
  55. package/dist/journey-cli/publish.js +70 -0
  56. package/dist/journey-cli/publish.js.map +1 -0
  57. package/dist/journey-cli/review.d.ts +4 -0
  58. package/dist/journey-cli/review.js +103 -0
  59. package/dist/journey-cli/review.js.map +1 -0
  60. package/dist/journey-cli/run.d.ts +4 -0
  61. package/dist/journey-cli/run.js +244 -0
  62. package/dist/journey-cli/run.js.map +1 -0
  63. package/dist/journey-cli/shared.d.ts +9 -0
  64. package/dist/journey-cli/shared.js +12 -0
  65. package/dist/journey-cli/shared.js.map +1 -0
  66. package/dist/journey-cli/verify.d.ts +4 -0
  67. package/dist/journey-cli/verify.js +125 -0
  68. package/dist/journey-cli/verify.js.map +1 -0
  69. package/dist/journey-cli.d.ts +3 -2
  70. package/dist/journey-cli.js +21 -949
  71. package/dist/journey-cli.js.map +1 -1
  72. package/dist/journey-review/render.d.ts +43 -0
  73. package/dist/journey-review/render.js +461 -0
  74. package/dist/journey-review/render.js.map +1 -0
  75. package/dist/journey-review.d.ts +1 -4
  76. package/dist/journey-review.js +16 -391
  77. package/dist/journey-review.js.map +1 -1
  78. package/dist/log-sources.js +17 -1
  79. package/dist/log-sources.js.map +1 -1
  80. package/dist/mcp-api.js +16 -821
  81. package/dist/mcp-api.js.map +1 -1
  82. package/dist/mcp-cli-tools.js +12 -4
  83. package/dist/mcp-cli-tools.js.map +1 -1
  84. package/dist/mcp-tools/cli.d.ts +3 -0
  85. package/dist/mcp-tools/cli.js +20 -0
  86. package/dist/mcp-tools/cli.js.map +1 -0
  87. package/dist/mcp-tools/explore.d.ts +3 -0
  88. package/dist/mcp-tools/explore.js +141 -0
  89. package/dist/mcp-tools/explore.js.map +1 -0
  90. package/dist/mcp-tools/findings.d.ts +3 -0
  91. package/dist/mcp-tools/findings.js +277 -0
  92. package/dist/mcp-tools/findings.js.map +1 -0
  93. package/dist/mcp-tools/inbox.d.ts +3 -0
  94. package/dist/mcp-tools/inbox.js +106 -0
  95. package/dist/mcp-tools/inbox.js.map +1 -0
  96. package/dist/mcp-tools/journey.d.ts +3 -0
  97. package/dist/mcp-tools/journey.js +260 -0
  98. package/dist/mcp-tools/journey.js.map +1 -0
  99. package/dist/mcp-tools/shared.d.ts +39 -0
  100. package/dist/mcp-tools/shared.js +69 -0
  101. package/dist/mcp-tools/shared.js.map +1 -0
  102. package/dist/sweep-api.d.ts +9 -0
  103. package/dist/sweep-api.js +70 -2
  104. package/dist/sweep-api.js.map +1 -1
  105. package/dist/sweep-cli.d.ts +2 -0
  106. package/dist/sweep-cli.js +22 -5
  107. package/dist/sweep-cli.js.map +1 -1
  108. package/dist/ux-usability/analysis.d.ts +14 -0
  109. package/dist/ux-usability/analysis.js +106 -0
  110. package/dist/ux-usability/analysis.js.map +1 -0
  111. package/dist/ux-usability/done-adjudication.d.ts +15 -0
  112. package/dist/ux-usability/done-adjudication.js +56 -0
  113. package/dist/ux-usability/done-adjudication.js.map +1 -0
  114. package/dist/ux-usability/hang.d.ts +5 -0
  115. package/dist/ux-usability/hang.js +41 -0
  116. package/dist/ux-usability/hang.js.map +1 -0
  117. package/dist/ux-usability/loop.d.ts +15 -0
  118. package/dist/ux-usability/loop.js +104 -0
  119. package/dist/ux-usability/loop.js.map +1 -0
  120. package/dist/ux-usability/observe.d.ts +7 -0
  121. package/dist/ux-usability/observe.js +89 -0
  122. package/dist/ux-usability/observe.js.map +1 -0
  123. package/dist/ux-usability/result.d.ts +11 -0
  124. package/dist/ux-usability/result.js +113 -0
  125. package/dist/ux-usability/result.js.map +1 -0
  126. package/dist/ux-usability/setup.d.ts +75 -0
  127. package/dist/ux-usability/setup.js +203 -0
  128. package/dist/ux-usability/setup.js.map +1 -0
  129. package/dist/ux-usability/teardown.d.ts +3 -0
  130. package/dist/ux-usability/teardown.js +13 -0
  131. package/dist/ux-usability/teardown.js.map +1 -0
  132. package/dist/ux-usability.d.ts +14 -17
  133. package/dist/ux-usability.js +15 -589
  134. package/dist/ux-usability.js.map +1 -1
  135. package/package.json +7 -7
@@ -1,55 +1,11 @@
1
- // ux-usability.ts — the live usability mission runner (`explore --strategy usability`) (#231).
2
- import { journeyFixtureBaseDir, resolveProjectMissionFixture } from "./journey-api.js";
3
- import { logsDirFor } from "./project-dir.js";
4
- import { mkdir, writeFile } from "node:fs/promises";
5
- import { join, resolve as resolvePath } from "node:path";
6
- import { PlaywrightBrowserPort } from "@jevitate/playwright";
7
- import { closeOnce, demoOverlayOf, finalizeVideos, runVideoDir, sessionLaunchOptions } from "./browser-run-options.js";
8
- import { runCaptureFor } from "./run-screenshots.js";
9
- import { evidenceOf, withRunEvidence } from "./defect-evidence.js";
10
- import { CastActor, BrowseTheWeb } from "@jevitate/screenplay";
11
- import { explore, runGoalBasedMission, assertAuthorizedExploreTarget, reproduceHang, hangFinding, hangOutcome, InvariantMonitor, BudgetMonitor, secretFieldSecrets, clippingSummary, detectClipping, detectOverflow, shouldCheckOverflow } from "@jevitate/explore";
12
- import { a11yChecks, analyzeClaims, buildReport, calibrationCaveat, claimsCaveat, detectFriction, detectSignals, groundFindings, loadV1Rubric, persistableScreen, resolveMinConfidence, resolveMaxFindingsPerRoute, resolveQualityPolicy, withSignalFindings, makeSignalFinding } from "@jevitate/ux";
13
- import { captureFindingShots, planGuardProbes, runGuardProbes, skippedProbes, withProbePage } from "./ux-claim-probe.js";
14
- import { NO_PRODUCT_FACTS_CAVEAT, loadProductFacts } from "./ux-product.js";
15
- import { conversationConfig } from "./conversation-options.js";
16
- import { loadUxMaxFindingsPerPage, loadUxMinConfidence, loadUxMinConfidenceByAppClass, loadUxShow } from "./ux-config.js";
17
- import { foldGoalOutcome, clock } from "@jevitate/domain";
18
- import { MissionJournal, artifactStamp, closeQuietly, resultPathFor, writeMissionResult } from "./mission-journal.js";
19
- import { MISSION_RESULT_SCHEMA_VERSION, advisoryDefects, defectFields } from "./result-schema.js";
20
- import { missionExitCode } from "./mission-exit.js";
21
- import { launchArmed } from "./launch-armed.js";
22
- import { branchFields, startFromJourney } from "./journey-prefix.js";
23
- import { failureWithHostStarved, finishHostHealth } from "./host-health-run.js";
24
- import { Http5xxOracle } from "@jevitate/explore";
25
- import { currentEngineInfo } from "./engine.js";
26
- import { openServerLogRuntime } from "./log-correlation.js";
27
- import { triageOf, serverLogRuntimeOptions } from "./explore-shared.js";
28
- import { assertSaveStorageStateOutsideProject, currentUrlSafe, persistStorageState, serverLogResult } from "./explore-api.js";
29
- import { transcriptPathFor } from "./transcript-file.js";
30
- import { UsabilityCapture } from "./usability-capture.js";
31
- import { DEFAULT_JUDGMENT_BUDGET, extractTypedValues, snapshotToEvidence } from "./ux-evidence.js";
32
- import { UxAnalysisFailedError } from "./ux-review.js";
33
- /** Usability reads only a spec's `budget` (#150) — never its `invariants`/`capture` (#86/#147, not supported here). */
34
- export class UsabilityInvariantsUnsupportedError extends Error {
35
- code = "E_USABILITY_INVARIANTS";
36
- constructor() {
37
- super("usability does not check app-declared invariants or captures — only a `budget` is read; " +
38
- "give a spec with an empty invariants array (and no capture) to use --invariants with usability");
39
- this.name = "UsabilityInvariantsUnsupportedError";
40
- }
41
- }
42
- /**
43
- * Opens a FRESH browser session for replays (hang reproduction): a new context from the same port
44
- * and options — same authenticated storageState, never the session the finding was made in.
45
- */
46
- function freshSessionOpener(portFactory, launch, allowlist) {
47
- return async () => {
48
- const session = await portFactory().open(launch);
49
- const actor = CastActor.named("replay").whoCan(new BrowseTheWeb(session, [...allowlist]));
50
- return { page: session.page, actor, close: () => session.close() };
51
- };
52
- }
1
+ import { UsabilityInvariantsUnsupportedError, setUpUsabilityMission } from "./ux-usability/setup.js";
2
+ import { runUsabilityLoop } from "./ux-usability/loop.js";
3
+ import { analyzeUsabilityRun } from "./ux-usability/analysis.js";
4
+ import { reproduceUsabilityHang } from "./ux-usability/hang.js";
5
+ import { adjudicateUsabilityJob } from "./ux-usability/done-adjudication.js";
6
+ import { writeUsabilityResult } from "./ux-usability/result.js";
7
+ import { tearDownUsabilityMission } from "./ux-usability/teardown.js";
8
+ export { UsabilityInvariantsUnsupportedError };
53
9
  /**
54
10
  * Live usability mission. Guardrail #1 authorizes BEFORE opening a browser.
55
11
  * Reuses explore()'s loop (goal = the job) and, via the additive onSnapshot
@@ -58,546 +14,16 @@ function freshSessionOpener(portFactory, launch, allowlist) {
58
14
  * ignored by explore). The temp profile dir is always removed.
59
15
  */
60
16
  export async function runUsabilityMission(opts) {
61
- const origin = assertAuthorizedExploreTarget(opts.url, opts.allowlist);
62
- assertSaveStorageStateOutsideProject(opts.saveStorageState);
63
- // #150 — usability reads only a spec's `budget`: it does not check invariants or captures (#86/
64
- // #147) today. Refused BEFORE a browser opens, same as every other bad-input refusal here.
65
- if (opts.invariants !== undefined && (opts.invariants.invariants.length > 0 || opts.invariants.capture !== undefined)) {
66
- throw new UsabilityInvariantsUnsupportedError();
67
- }
68
- // Validate the cutoff before a browser opens — a bad value fails fast, never mid-run.
69
- const minConfidence = resolveMinConfidence(opts.minConfidence, opts.env ?? process.env, loadUxMinConfidence(opts.configPath), loadUxMinConfidenceByAppClass(opts.configPath, opts.appContext.appClass));
70
- const quality = resolveQualityPolicy(opts.show, opts.env ?? process.env, loadUxShow(opts.configPath), opts.appContext.appClass);
71
- const maxFindingsPerRoute = resolveMaxFindingsPerRoute(opts.maxFindingsPerRoute, opts.env ?? process.env, loadUxMaxFindingsPerPage(opts.configPath));
72
- const fixture = opts.fixture === undefined ? undefined : await resolveProjectMissionFixture(opts.fixture);
73
- // #198: the product facts are validated before a browser opens (a bad file is a usage error).
74
- const product = await loadProductFacts(opts.product);
75
- // #250/#251: a recorded or screenshotted run's sessions carry the live pixel mask from their first
76
- // paint; `--screenshots` captures after each step (the Recording path names their folder).
77
- const runCapture = runCaptureFor({
78
- recordsVideo: opts.browser?.recordVideo !== undefined,
79
- screenshots: opts.screenshots,
80
- secrets: [...(opts.secrets ?? []), ...secretFieldSecrets(opts.secretFields)],
81
- artifactPath: () => journal.recordingPath,
82
- title: `usability: ${opts.job}`,
83
- });
84
- const portFactory = runCapture.wrap(opts.browserPortFactory ?? (() => new PlaywrightBrowserPort()));
85
- const port = portFactory();
86
- const collected = [];
87
- const history = [];
88
- // #149: one signal finding per distinct fingerprint (route + element) — a wide table seen across
89
- // many observed screens is still ONE finding, never a finding per occurrence.
90
- const overflowFindings = [];
91
- const seenOverflow = new Set();
92
- const extract = opts.extractText ??
93
- (async (s) => s.page.evaluate(() => (typeof document !== "undefined" && document.body ? document.body.innerText : "")));
94
- const outDir = opts.outDir ?? logsDirFor();
95
- const iso = (opts.nowIso ?? (() => clock.nowIso()))();
96
- const stamp = artifactStamp(iso);
97
- const reportPath = join(outDir, `usability-${stamp}.json`);
98
- // #98 — the same artifact shape as goal/adversarial missions, next to the report: the decision
99
- // transcript (each step's redacted typed value and screenshot), the Recording and the per-step
100
- // screenshots. Crash-safe: the transcript and partial Recording are flushed after every step.
101
- const journal = new MissionJournal(join(outDir, `usability-${stamp}.recording.json`), transcriptPathFor(reportPath));
102
- const screenshotDir = join(outDir, `usability-${stamp}.screens`);
103
- // #245: shown/recorded like every other strategy's session (videos in `usability-<stamp>.videos/`).
104
- const videoDir = runVideoDir(opts.browser, reportPath);
105
- const launch = {
106
- ...sessionLaunchOptions(opts.browser, videoDir),
107
- allowedOrigins: [...opts.allowlist],
108
- baseUrl: origin,
109
- ...opts.emulation,
110
- ...(opts.storageState !== undefined ? { storageState: opts.storageState } : {}),
111
- };
112
- // A bound secret (or TOTP seed) is a run secret too: masked on screen, redacted everywhere.
113
- const secrets = [...(opts.secrets ?? []), ...secretFieldSecrets(opts.secretFields)];
114
- // #163: this run's own share of a (possibly shared) tracker: its usage and sidecar.
115
- const runUsage = opts.usage?.scope();
116
- // The screenshot capture needs the open page; until then a killed run has taken none.
117
- let armedCapture;
118
- // Crash-safe on SIGTERM/SIGINT too (#94): a partial `inconclusive` result is written from
119
- // whatever the journal has already flushed, and the process exits with the conventional code.
120
- // #120: the transcript lives next to the REPORT (`usability-<stamp>.transcript.json`), not the
121
- // Recording — so the killed run's result names the real file, and reports the live step list,
122
- // the tokens spent so far and the screens already observed.
123
- // #226: the kill switch is armed BEFORE the host sampler and the browser launch (see launch-armed.ts).
124
- const { disarmKillSwitch, health, session, snapshotter } = await launchArmed({
125
- hostHealth: opts.hostHealth,
126
- saveStorageState: opts.saveStorageState !== undefined,
127
- open: () => port.open(launch),
128
- mission: (hooks) => ({
129
- // #220: the killed run's partial result carries the unified schema's common fields too.
130
- strategy: "usability",
131
- target: { seedUrl: opts.url, allowlist: [...opts.allowlist], ...(opts.storageState !== undefined ? { storageStatePath: resolvePath(opts.storageState) } : {}) },
132
- recordingPath: journal.recordingPath,
133
- hostHealth: hooks.hostHealth,
134
- ...(videoDir === undefined ? {} : { videoDir }),
135
- transcriptPath: journal.transcriptPath,
136
- transcript: () => journal.transcript,
137
- ...(runUsage === undefined ? {} : { usage: runUsage }),
138
- partialReport: () => ({ screensObserved: collected.length, screenshotDir, screenshots: armedCapture === undefined ? [] : armedCapture.screenshots() }),
139
- ...(opts.saveStorageState === undefined ? {} : { storageState: { path: opts.saveStorageState, snapshot: hooks.snapshot } }),
140
- }),
141
- });
142
- // #324: the engine runs a cmd: binding's command at type time. A value read mid-run joins this
143
- // run's secrets at once (screenshots' scan, result, evidence, report) and — #360 — the run's pixel
144
- // mask, before the value is typed. (It was once handed to UsabilityCapture, which has no such
145
- // option, so a usability run could not type a cmd: field at all.)
146
- const uxSecretCommand = opts.secretCommand === undefined
147
- ? undefined
148
- : async (command) => {
149
- const out = await opts.secretCommand(command);
150
- const value = out.trim();
151
- if (value !== "" && !secrets.includes(value))
152
- secrets.push(value);
153
- if (value !== "")
154
- await runCapture.mask.addSecret(value);
155
- return out;
156
- };
157
- // #208: the shared HTTP 5xx hard signal, listening from before the first navigation.
158
- const http5xx = new Http5xxOracle(session.page, { allowlist: opts.allowlist });
159
- const capture = new UsabilityCapture({
160
- page: session.page,
161
- screenshotDir,
162
- secrets,
163
- ...(opts.secretFields === undefined ? {} : { secretFields: opts.secretFields }),
164
- });
165
- armedCapture = capture;
166
- // The usability capture (screenshots) and the journal (crash-safe flush) are the EXISTING listener
167
- // chain; a server-log runtime (#142) is inserted in FRONT of it (never replacing it) so every step
168
- // still gets its screenshot/flush exactly as before, whether or not --log-source was given. #159:
169
- // every settled step also refreshes the in-memory storageState snapshot (a cheap no-op when
170
- // `--save-storage-state` was not given).
171
- const journalListener = (entry, all) => {
172
- runCapture.noteEntry(session.page, entry);
173
- health.noteStep(entry);
174
- http5xx.noteStep(entry);
175
- capture.noteEntry(entry, all);
176
- journal.onTranscriptEntry(entry, capture.withScreenshots(all));
177
- snapshotter.noteSettledStep(currentUrlSafe(session));
178
- };
179
- const serverLog = openServerLogRuntime({
180
- ...serverLogRuntimeOptions(opts.serverLog),
181
- secrets,
182
- onTranscriptEntry: journalListener,
183
- });
184
- // #204: every request's correlation ids, from before the first navigation.
185
- serverLog?.observe(session.page);
186
- // #159/#245: persisted and closed once — early (before the result is written) when recording video.
187
- const closeSession = closeOnce(async () => {
188
- await persistStorageState(session, opts.saveStorageState, snapshotter);
189
- await closeQuietly(session);
190
- });
17
+ const ctx = await setUpUsabilityMission(opts);
191
18
  try {
192
- // #293: a journey-anchored review first replays its Journey's prefix into this very session.
193
- const start = await startFromJourney(opts.journeyPrefix, session, opts.url, opts.allowlist, opts.browser);
194
- const actor = CastActor.named("usability-mission").whoCan(new BrowseTheWeb(session, [...opts.allowlist]));
195
- // #150 — usability's own budget wiring: a plain `InvariantMonitor` reads a budget's declared
196
- // observables (the same #86/#135 read/auth/redaction machinery), but this mission folds NO
197
- // invariant defects — only a budget crossing can end the run early, via the same pre-action
198
- // guard / post-settle hooks `explore()` offers every mission.
199
- const budgetDecls = opts.invariants?.budget ?? [];
200
- const invariantMonitor = opts.invariants === undefined || budgetDecls.length === 0
201
- ? null
202
- : new InvariantMonitor(opts.invariants, {
203
- allowlist: opts.allowlist,
204
- baseUrl: opts.url,
205
- ...(opts.secrets === undefined ? {} : { secrets: opts.secrets }),
206
- ...(opts.invariantAuthTokens === undefined ? {} : { authTokens: opts.invariantAuthTokens }),
207
- });
208
- const budget = invariantMonitor === null ? null : new BudgetMonitor(budgetDecls, invariantMonitor);
209
- let budgetSettledSteps = 0;
210
- const exploreCfg = {
211
- ...(opts.target?.timing === undefined ? {} : { timingConfig: opts.target.timing }),
212
- ...(opts.target?.settle === undefined ? {} : { settle: opts.target.settle }),
213
- ...(opts.target?.hangs === undefined ? {} : { hangs: opts.target.hangs }),
214
- ...(opts.target?.safety === undefined ? {} : { safety: opts.target.safety }),
215
- onTranscriptEntry: serverLog?.onTranscriptEntry ?? journalListener,
216
- onRecording: journal.onRecording,
217
- // #303 (opt-in): action deltas, with Jev's advisory relevance labels.
218
- ...(opts.actionDeltas === true ? { actionDeltas: { jev: true } } : {}),
219
- hostHealth: health,
220
- demoOverlay: demoOverlayOf(opts.browser),
221
- ...(opts.secretFields === undefined ? {} : { secretFields: opts.secretFields }),
222
- ...(uxSecretCommand === undefined ? {} : { secretCommand: uxSecretCommand }),
223
- ...(opts.secretCommandAttempts === undefined ? {} : { secretCommandAttempts: opts.secretCommandAttempts }),
224
- actor,
225
- judge: opts.judge,
226
- gen: opts.gen,
227
- goal: opts.job,
228
- allowlist: opts.allowlist,
229
- startUrl: start.url,
230
- ...(start.branch === undefined ? {} : { startInPlace: true }),
231
- ...(start.restart === undefined ? {} : { restartAtStart: start.restart.restartAtStart }),
232
- bounds: opts.bounds,
233
- secrets: opts.secrets,
234
- site: origin,
235
- fixture,
236
- fixtureRoot: journeyFixtureBaseDir(),
237
- ...conversationConfig(opts.conversation),
238
- onSnapshot: async (snap) => {
239
- let visibleText = "";
240
- try {
241
- visibleText = await extract(session);
242
- }
243
- catch {
244
- visibleText = ""; // best-effort; the analyzer Skips visibleText items honestly
245
- }
246
- const ev = snapshotToEvidence(snap, visibleText, opts.appContext, opts.job, [...history]);
247
- collected.push(ev);
248
- history.push({ screenId: ev.screenId, url: ev.url });
249
- // #98 the step's (secret-masked) screenshot; #96 the screen's facts for the signal oracles.
250
- await capture.observe(snap, visibleText);
251
- // #149: horizontal-overflow hard signal — pure DOM geometry, never a model judgment.
252
- // Best-effort like visibleText extraction above: a detection failure never fails the mission.
253
- try {
254
- const vp = session.page.viewportSize();
255
- if (shouldCheckOverflow(vp?.width, opts.overflow?.checkOverflow ?? false)) {
256
- const overflow = await detectOverflow(session.page, {
257
- viewport: vp ?? { width: 1280, height: 720 },
258
- ...(opts.emulation?.device === undefined ? {} : { device: opts.emulation.device }),
259
- ...(opts.overflow?.toleranceCss === undefined ? {} : { toleranceCss: opts.overflow.toleranceCss }),
260
- ...(opts.overflow?.ignoreSelectors === undefined ? {} : { ignoreSelectors: opts.overflow.ignoreSelectors }),
261
- secrets,
262
- });
263
- if (overflow !== null && !seenOverflow.has(overflow.fingerprint)) {
264
- seenOverflow.add(overflow.fingerprint);
265
- overflowFindings.push(makeSignalFinding({
266
- kind: "horizontal-overflow",
267
- confidence: 0.9,
268
- url: ev.url,
269
- screenId: ev.screenId,
270
- observation: `${overflow.element.descriptor} overflows the ${overflow.viewport.width}px viewport by ${overflow.overflowPx}px on ${overflow.route}.`,
271
- userImpact: "Content extends past the visible viewport; a user on this device must discover and use horizontal scrolling to see it, and may miss it entirely.",
272
- recommendation: `Constrain ${overflow.element.descriptor} to the viewport width (e.g. a responsive layout, or an explicit scroll container) at ${overflow.viewport.width}px.`,
273
- controls: [overflow.element.descriptor],
274
- evidence: {
275
- kind: "horizontal-overflow",
276
- steps: [history.length],
277
- requests: [],
278
- detail: `scrollWidth exceeds innerWidth by ${overflow.overflowPx}px at a ${overflow.viewport.width}x${overflow.viewport.height} viewport`,
279
- },
280
- }));
281
- }
282
- // #302: text cut off vertically (a fixed-height box, or above the page top) — one finding per element.
283
- const clipped = await detectClipping(session.page, {
284
- viewport: vp ?? { width: 1280, height: 720 },
285
- ...(opts.emulation?.device === undefined ? {} : { device: opts.emulation.device }),
286
- ...(opts.overflow?.ignoreSelectors === undefined ? {} : { ignoreSelectors: opts.overflow.ignoreSelectors }),
287
- secrets,
288
- });
289
- for (const c of clipped) {
290
- if (seenOverflow.has(c.fingerprint))
291
- continue;
292
- seenOverflow.add(c.fingerprint);
293
- overflowFindings.push(makeSignalFinding({
294
- kind: "vertical-clipping",
295
- confidence: 0.9,
296
- url: ev.url,
297
- screenId: ev.screenId,
298
- observation: c.cause === "overflow-hidden"
299
- ? `${c.element.descriptor} cuts off its text by ${c.clippedPx}px on ${c.route}: the content is taller than the box and overflow is hidden.`
300
- : `${c.element.descriptor} is cut off ${c.clippedPx}px above the top of the page on ${c.route}.`,
301
- userImpact: "Part of the text is cut off and no scroll position shows it: a user on this device cannot read it.",
302
- recommendation: c.cause === "overflow-hidden"
303
- ? `Let ${c.element.descriptor} grow with its content (min-height instead of height), make it scrollable, or truncate on purpose with line-clamp at ${c.viewport.width}px.`
304
- : `Give the container of ${c.element.descriptor} room for its wrapped content (no fixed height, or no wrapping) at ${c.viewport.width}px.`,
305
- controls: [c.element.descriptor],
306
- evidence: { kind: "vertical-clipping", steps: [history.length], requests: [], detail: clippingSummary(c) },
307
- }));
308
- }
309
- }
310
- }
311
- catch {
312
- // Best-effort: the run's own explore loop is never held up or failed by this check.
313
- }
314
- },
315
- ...(budget === null
316
- ? {}
317
- : {
318
- onBeforeAction: async (info) => {
319
- const g = await budget.guard(session.page, info);
320
- return g.refuse ? { refuse: true, reason: g.reason ?? "budget guard refused the action" } : { refuse: false };
321
- },
322
- onSettled: async () => {
323
- budgetSettledSteps += 1;
324
- // The FIRST settled snapshot (before any action) is the budget's baseline.
325
- if (budgetSettledSteps === 1) {
326
- const b = await budget.baseline(session.page);
327
- return b.crossed ? { stop: true, reason: b.reason ?? "budget observable unreadable at run start" } : { stop: false };
328
- }
329
- const r = await budget.afterSettle(session.page, budgetSettledSteps);
330
- return r.crossed ? { stop: true, reason: r.reason ?? "mission budget crossed" } : { stop: false };
331
- },
332
- }),
333
- };
334
- const brief = "usability review: pursue the stated job as a plausible first-time user, using only what is on screen";
335
- // #225: `--success` is never ignored — with checks the loop runs through the goal mission's
336
- // independent adjudication (the same oracle, `held`/vacuous rules and verdict as a goal run).
337
- const checks = opts.successChecks ?? [];
338
- const adjudication = checks.length === 0
339
- ? undefined
340
- : await runGoalBasedMission({
341
- ...exploreCfg,
342
- missionBrief: brief,
343
- successChecks: checks,
344
- // #225: a `done` whose check failed on a job judged done ends the run — never the whole budget.
345
- stopWhenJudgedDone: true,
346
- ...(opts.successWhen === undefined ? {} : { successWhen: opts.successWhen }),
347
- ...(opts.allowVacuousChecks === true ? { allowVacuousChecks: true } : {}),
348
- });
349
- const run = adjudication?.run ?? (await explore({ ...exploreCfg, missionContext: brief }));
350
- // Never blocks the mission itself: the drain wait happens AFTER `explore()` returned.
351
- const serverLogRun = serverLog === undefined ? undefined : await serverLog.finish(run.transcript);
352
- journal.writeRecording(run.recording);
353
- journal.writeTranscript(capture.withScreenshots(serverLogRun?.transcript ?? run.transcript));
354
- // #85 item 1: the live run's own typed values (from its emitted Recording's fill/select
355
- // steps — never a secret), so the vocabulary/jargon tier can tell the app's own copy apart
356
- // from user-authored content it merely echoed back (same mechanism as the offline pass).
357
- const typedValues = extractTypedValues(run.recording);
358
- const screens = typedValues.length > 0 ? collected.map((ev) => ({ ...ev, typedValues })) : collected;
359
- // #96: findings from the run's own measurements (hung request, duplicate write, internal id,
360
- // inert control) — independent code, no model — reported alongside the rubric's.
361
- const signalCapture = await capture.signalCapture(run.transcript, typedValues);
362
- // A gRPC-web/Connect read is never a duplicate write (#110); `--read-rpc` marks more reads.
363
- const readRequests = opts.target?.safety?.readRequests;
364
- const signalFindings = [
365
- ...detectSignals(signalCapture, readRequests === undefined ? opts.signals : { ...opts.signals, readRequests }),
366
- // #149: horizontal-overflow, computed live during the run (never from the captured timeline).
367
- ...overflowFindings,
368
- ];
369
- // #132: the friction the run walked into — what grounds (or not) each rubric finding.
370
- const friction = detectFriction(signalCapture, run.outcome);
371
- // #198: every destructive control on an analyzed screen is clicked once with its writes blocked
372
- // (ux-claim-probe.ts) — the code evidence `destructive-unguarded` claims are verified against.
373
- // On a dedicated page in the run's context, after the loop: the run's oracles never see it.
374
- // A screen that cannot be redacted means no probes at all (destructive claims: unverifiable).
375
- let probes;
376
- try {
377
- const plan = planGuardProbes(screens, secrets, opts.target?.safety);
378
- probes = [...plan.refused];
379
- // Clicking is opt-in (--probe-guards): without it every destructive control is `skipped`, and
380
- // its guard claim is reported unverifiable — never asserted, never silently dropped.
381
- if (opts.probeGuards !== true)
382
- probes.push(...skippedProbes(plan.targets));
383
- else if (plan.targets.length > 0) {
384
- try {
385
- probes.push(...(await runGuardProbes(session.page, plan.targets, { allowlist: opts.allowlist, secrets })));
386
- }
387
- catch (err) {
388
- const why = `the guard probe could not open a page: ${err instanceof Error ? err.message.split("\n")[0] : String(err)}`;
389
- probes.push(...plan.targets.map((t) => ({ screenId: t.screen.screenId, route: t.route, control: t.label, controlKey: t.key, status: "failed", detail: why })));
390
- }
391
- }
392
- }
393
- catch {
394
- probes = undefined;
395
- }
396
- // #134: the evidence sidecar, written through the redaction door BEFORE analysis (so it exists
397
- // even when analysis fails). Fail-closed: if any screen cannot be redacted, no file is written.
398
- const evidencePath = join(outDir, `usability-${stamp}.evidence.json`);
399
- let evidenceWritten = null;
400
- try {
401
- const file = {
402
- version: 1,
403
- appContext: opts.appContext,
404
- job: opts.job,
405
- screens: screens.map((ev) => persistableScreen(ev, secrets)),
406
- signals: signalCapture,
407
- outcome: run.outcome,
408
- ...(probes === undefined ? {} : { probes }),
409
- };
410
- await mkdir(outDir, { recursive: true });
411
- await writeFile(evidencePath, `${JSON.stringify(file, null, 2)}\n`, "utf8");
412
- evidenceWritten = evidencePath;
413
- }
414
- catch {
415
- evidenceWritten = null;
416
- }
417
- // #198: findings are claims verified by code (claims.ts) — guard probes, product facts and the
418
- // friction the run walked into — categorized and graded by Jev, written from templates.
419
- const claimed = await analyzeClaims({
420
- screens,
421
- rubric: loadV1Rubric(),
422
- appContext: opts.appContext,
423
- secrets,
424
- judgmentBudget: opts.judgmentBudget ?? DEFAULT_JUDGMENT_BUDGET,
425
- friction,
426
- steps: signalCapture.steps,
427
- signalFindings,
428
- ...(probes === undefined ? {} : { probes }),
429
- ...(product.facts === undefined ? {} : { facts: product.facts }),
430
- }, { judge: opts.judge, gen: opts.gen, a11yChecker: a11yChecks, ...(opts.polish === true ? { polish: true } : {}) });
431
- // #198: a cropped, masked screenshot with the cited control boxed, per verified claim finding.
432
- let outcome = claimed;
433
- if (claimed.kind === "analyzed" && claimed.findings.some((f) => f.claim !== undefined)) {
434
- const byScreen = new Map(screens.map((ev) => [ev.screenId, ev]));
435
- const dir = join(outDir, `usability-${stamp}.findings`);
436
- try {
437
- const findings = await withProbePage(session.page, (p) => captureFindingShots(p, claimed.findings, byScreen, { dir, allowlist: opts.allowlist, secrets }));
438
- outcome = { ...claimed, findings };
439
- }
440
- catch {
441
- outcome = claimed; // presentation only: findings without screenshots are still the findings
442
- }
443
- }
444
- // #126: a run that stops on a hang is never `clean` — it is reproduced in fresh contexts (same
445
- // as a goal mission) and mapped through the same hang/intermittent/inconclusive rule.
446
- let hang;
447
- if (run.stop === "hang" && run.hang !== undefined) {
448
- const h = run.hang;
449
- const reproduction = await reproduceHang({
450
- recording: run.recording,
451
- recordingStepIndex: h.recordingStepIndex,
452
- hang: h.signal,
453
- openSession: freshSessionOpener(portFactory, launch, opts.allowlist),
454
- fixtureBaseDir: journeyFixtureBaseDir(),
455
- ...(opts.target?.safety === undefined ? {} : { safety: opts.target.safety }),
456
- writtenBy: run.sideEffects.map((e) => e.control), // #181
457
- perceive: {
458
- ...(opts.target?.settle === undefined ? {} : { settleConfig: opts.target.settle }),
459
- ...(opts.target?.hangs === undefined ? {} : { hangConfig: opts.target.hangs }),
460
- },
461
- });
462
- hang = hangFinding(h.signal, run.transcript, h.recordingStepIndex, reproduction);
463
- }
464
- // #150 — a declared mission spend budget crossed (or a paid action was refused before crossing
465
- // it): a clean, deliberate stop, never `crashed` — but never `clean` either (the run's own work
466
- // past the stop is unproven), so it maps to `inconclusive` the same as `run.stop === "inconclusive"`.
467
- const loopOutcome = run.stop === "crashed"
468
- ? "crashed"
469
- : run.stop === "inconclusive" || run.stop === "budget"
470
- ? "inconclusive"
471
- : run.stop === "hang"
472
- ? hangOutcome(hang?.reproduction.status ?? "inconclusive")
473
- : "clean";
474
- // #209: a review whose job was never completed (the loop gave up, ran out of budget, or its
475
- // `done` was never verified) did not see what a user who finished it would: its silence about
476
- // the rest proves nothing, so it is `inconclusive` (`job-incomplete`) — never `clean`, whatever
477
- // the advisory UX findings say.
478
- const jobIncomplete = loopOutcome === "clean" && run.outcome.status === "incomplete"
479
- ? { kind: "job-incomplete", message: `the job under review was not completed: ${run.outcome.reason}` }
480
- : undefined;
481
- // #225: with success checks, THEY decide whether the job was done — folded exactly like a goal
482
- // run's ending (a failed check `defects-found`, a vacuous one `inconclusive`, all held `clean`).
483
- const checked = adjudication === undefined ? undefined : foldGoalOutcome(adjudication.outcome);
484
- const jobVerdict = loopOutcome !== "clean" ? loopOutcome : checked !== undefined ? checked : jobIncomplete === undefined ? "clean" : "inconclusive";
485
- // #225/#209: a failed success check is named as such (never "job-incomplete"), as on a goal run.
486
- const failedChecks = adjudication?.checks.filter((c) => !c.passed) ?? [];
487
- const checkFailure = adjudication?.failure ??
488
- (failedChecks.length === 0
489
- ? undefined
490
- : { kind: "success-check-failed", message: `success check ${failedChecks.map((c) => `'${c.check}' ${c.detail}`).join("; ")}` });
491
- const jobFailure = loopOutcome !== "clean" ? undefined : checked === undefined ? jobIncomplete : checked === "clean" ? undefined : (checkFailure ?? jobIncomplete);
492
- // #203: most steps on a starved host → `inconclusive` (degraded-environment), never `clean`.
493
- // #213: a job whose completion code verified (#225: its success checks held, or its done was
494
- // adjudicated from save signals) is a positive proof — a starved host does not undo it. Only an
495
- // unverified ending is downgraded, and it keeps its own reason (a failed check) in the degraded one.
496
- // A model-only `grounded-judgment` is not code's proof, so it is still downgraded.
497
- const jobVerified = jobVerdict === "clean" &&
498
- (checked === "clean" || (checked === undefined && run.outcome.status === "completed" && run.outcome.verifiedBy !== "grounded-judgment"));
499
- const host = await finishHostHealth(health, jobVerdict, {
500
- verified: jobVerified,
501
- ...(run.failure === undefined ? {} : { failure: run.failure }),
502
- ...(jobFailure === undefined ? {} : { wouldHaveBeen: jobFailure.message }),
503
- });
504
- const runOutcome = host.outcome;
505
- // #245: every context closed (videos finalized) before the result naming them is written.
506
- const shotFields = await runCapture.finish();
507
- const videos = await finalizeVideos(videoDir, closeSession);
508
- const base = {
509
- schemaVersion: MISSION_RESULT_SCHEMA_VERSION,
510
- strategy: "usability",
511
- target: {
512
- seedUrl: start.persistUrl,
513
- allowlist: [...opts.allowlist],
514
- ...(opts.storageState !== undefined ? { storageStatePath: resolvePath(opts.storageState) } : {}),
515
- },
516
- ...branchFields(start),
517
- recordingPaths: [journal.recordingPath],
518
- ...videos,
519
- ...shotFields,
520
- resultPath: resultPathFor(journal.recordingPath),
521
- hangs: hang === undefined ? [] : [hang],
522
- timing: run.timing,
523
- stop: run.stop,
524
- outcome: run.outcome,
525
- ...(run.answer === undefined ? {} : { answer: run.answer }),
526
- screensObserved: collected.length,
527
- transcriptPath: journal.transcriptPath,
528
- screenshotDir,
529
- evidencePath: evidenceWritten,
530
- screenshots: capture.screenshots(),
531
- sideEffects: run.sideEffects,
532
- ...(run.sideEffectsTruncated === undefined ? {} : { sideEffectsTruncated: run.sideEffectsTruncated }),
533
- ...(run.safetyOverrides === undefined ? {} : { safetyOverrides: run.safetyOverrides }),
534
- ...(run.actionDeltas === undefined ? {} : { actionDeltas: run.actionDeltas }),
535
- engine: currentEngineInfo(),
536
- ...(() => {
537
- const f = failureWithHostStarved(host, run.failure ?? host.failure ?? jobFailure);
538
- return f === undefined ? {} : { failure: f };
539
- })(),
540
- ...(run.crash === undefined ? {} : { crash: run.crash }),
541
- finalUrl: run.finalUrl,
542
- decisions: run.decisions,
543
- actions: run.actions,
544
- depth: run.depth,
545
- ...(runUsage === undefined ? {} : { usage: runUsage.snapshot() }),
546
- ...(hang === undefined ? {} : { hang }),
547
- // #142 follow-up: reported but never gates `missionOutcome`/`exitCode` — a UX finding is
548
- // always advisory, and a `server-log` defect here is treated the same way. So is an HTTP 5xx
549
- // hard-signal defect (#208): listed with its fingerprint (verify-fix replays it), advisory here.
550
- ...serverLogResult(serverLogRun),
551
- ...defectFields([
552
- ...http5xx.defects(run.transcript, run.recording.pages.flatMap((p) => p.steps)[0]?.step.kind === "navigate" ? 1 : 0).map((d) => ({ ...d, advisory: true })),
553
- ...advisoryDefects(serverLogRun?.defects),
554
- ]),
555
- ...(budget === null ? {} : { budget: budget.trajectory() }),
556
- ...(adjudication === undefined
557
- ? {}
558
- : {
559
- goalOutcome: adjudication.outcome,
560
- checks: adjudication.checks,
561
- ...(adjudication.warnings === undefined ? {} : { checkWarnings: adjudication.warnings }),
562
- }),
563
- ...host.fields,
564
- };
565
- if (outcome.kind === "failed") {
566
- // The analysis is the review's product: without it the review is inconclusive (never a
567
- // fabricated clean report) — but the run's transcript is kept, and this is a typed result.
568
- const why = new UxAnalysisFailedError(outcome.reason, outcome.screenId, outcome.rubricItemId).message;
569
- const missionOutcome = runOutcome === "clean" ? "inconclusive" : runOutcome;
570
- const unavailable = {
571
- ...base,
572
- report: null,
573
- reportPath: null,
574
- missionOutcome,
575
- exitCode: missionExitCode(missionOutcome),
576
- analysisUnavailable: why,
577
- };
578
- // Persisted like every other mission's typed result, so MCP `get_mission_result` can read it (#117).
579
- return await withRunEvidence({ ...unavailable, resultPath: writeMissionResult(journal.recordingPath, missionOutcome, unavailable.exitCode, unavailable, runUsage) }, evidenceOf(opts, secrets), triageOf(opts, serverLogRun, secrets));
580
- }
581
- const report = buildReport(groundFindings(withSignalFindings(outcome, signalFindings), friction), {
582
- minConfidence,
583
- quality,
584
- maxFindingsPerRoute,
585
- ...(product.facts === undefined ? { evidenceCaveats: [NO_PRODUCT_FACTS_CAVEAT] } : {}),
586
- calibrationCaveats: [calibrationCaveat(opts.appContext.appClass), claimsCaveat()],
587
- });
588
- await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
589
- const reviewed = { ...base, report, reportPath, missionOutcome: runOutcome, exitCode: missionExitCode(runOutcome) };
590
- return await withRunEvidence({ ...reviewed, resultPath: writeMissionResult(journal.recordingPath, runOutcome, reviewed.exitCode, reviewed, runUsage) }, evidenceOf(opts, secrets), triageOf(opts, serverLogRun, secrets));
19
+ const loop = await runUsabilityLoop(ctx);
20
+ const analysis = await analyzeUsabilityRun(ctx, loop);
21
+ const hang = await reproduceUsabilityHang(ctx, loop.run);
22
+ const verdict = await adjudicateUsabilityJob(ctx, loop, hang);
23
+ return await writeUsabilityResult(ctx, loop, analysis, hang, verdict);
591
24
  }
592
25
  finally {
593
- capture.detach();
594
- disarmKillSwitch();
595
- health.stop();
596
- // Safety net: if the mission threw before `serverLog.finish()` ran, close sources immediately
597
- // (no drain wait) rather than leaving them open until process exit.
598
- await serverLog?.abort();
599
- // #159: reaches this even when the mission above threw — the context is still open here.
600
- await closeSession();
26
+ await tearDownUsabilityMission(ctx);
601
27
  }
602
28
  }
603
29
  //# sourceMappingURL=ux-usability.js.map