@revyl/verify 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/LICENSE +201 -0
- package/README.md +137 -2
- package/SECURITY.md +13 -0
- package/dist/api/client.d.ts +63 -0
- package/dist/api/client.js +201 -0
- package/dist/api/types.d.ts +222 -0
- package/dist/api/types.js +1 -0
- package/dist/auth.d.ts +17 -0
- package/dist/auth.js +65 -0
- package/dist/cleanup.d.ts +13 -0
- package/dist/cleanup.js +72 -0
- package/dist/cli/init.d.ts +14 -0
- package/dist/cli/init.js +115 -0
- package/dist/cli/main.d.ts +3 -0
- package/dist/cli/main.js +127 -0
- package/dist/computer-session.d.ts +43 -0
- package/dist/computer-session.js +418 -0
- package/dist/config.d.ts +88 -0
- package/dist/config.js +212 -0
- package/dist/errors.d.ts +9 -0
- package/dist/errors.js +14 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.js +3 -0
- package/dist/open.d.ts +13 -0
- package/dist/open.js +33 -0
- package/dist/screen-readiness.d.ts +1 -0
- package/dist/screen-readiness.js +27 -0
- package/dist/session.d.ts +153 -0
- package/dist/session.js +443 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.js +6 -0
- package/docs/README.md +19 -0
- package/docs/agents.md +43 -0
- package/docs/api.md +120 -0
- package/docs/ci.md +70 -0
- package/docs/computers.md +83 -0
- package/docs/configuration.md +92 -0
- package/docs/end-to-end.md +105 -0
- package/docs/errors.md +39 -0
- package/docs/getting-started.md +88 -0
- package/docs/linux-computers-contract.md +53 -0
- package/docs/llms.txt +19 -0
- package/docs/mobile.md +49 -0
- package/docs/web.md +37 -0
- package/package.json +71 -4
package/dist/session.js
ADDED
|
@@ -0,0 +1,443 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { dirname, resolve } from 'node:path';
|
|
4
|
+
import { setTimeout as delay } from 'node:timers/promises';
|
|
5
|
+
import { apiError, parseJson, responseDetail, responseErrorCode, sessionViewerUrl, } from './api/client.js';
|
|
6
|
+
import { trackSession, untrackSession } from './cleanup.js';
|
|
7
|
+
import { VerifyError, errorMessage, isVerifyError } from './errors.js';
|
|
8
|
+
import { screenHasContent } from './screen-readiness.js';
|
|
9
|
+
const POLL_INTERVAL_MS = 2_000;
|
|
10
|
+
const DEVICE_CONNECT_TIMEOUT_MS = 90_000;
|
|
11
|
+
const STEP_POLL_BASE_MS = 500;
|
|
12
|
+
const STEP_POLL_MAX_MS = 5_000;
|
|
13
|
+
const STEP_CANCEL_TIMEOUT_MS = 5_000;
|
|
14
|
+
const MAX_STEP_TEXT_LENGTH = 4_000;
|
|
15
|
+
const ENDED_STATUSES = new Set(['completed', 'failed', 'cancelled', 'timeout', 'stopping']);
|
|
16
|
+
const STEP_TERMINAL_STATUSES = new Set(['completed', 'failed']);
|
|
17
|
+
const SESSION_NOT_RUNNING = /session is (not running|not active|stopping)/i;
|
|
18
|
+
const UNCERTAIN_ERROR_CODES = new Set(['request_timeout', 'connection_closed', 'protocol_error']);
|
|
19
|
+
/** A Revyl cloud browser or iOS or Android device. */
|
|
20
|
+
export class DeviceSession {
|
|
21
|
+
client;
|
|
22
|
+
options;
|
|
23
|
+
platform;
|
|
24
|
+
/** The target name: its key in verify.config.ts, or the platform for a target passed inline. */
|
|
25
|
+
target;
|
|
26
|
+
/** Revyl's session id. */
|
|
27
|
+
id;
|
|
28
|
+
/** The live view of the session in Revyl, and its recording and report once it ends. */
|
|
29
|
+
viewerUrl;
|
|
30
|
+
/** Every instruction and validation run on this session, in order. */
|
|
31
|
+
steps = [];
|
|
32
|
+
workflowRunId;
|
|
33
|
+
closePromise;
|
|
34
|
+
stepRunning = false;
|
|
35
|
+
constructor(client, target, id, workflowRunId, options) {
|
|
36
|
+
this.client = client;
|
|
37
|
+
this.options = options;
|
|
38
|
+
this.platform = target.platform;
|
|
39
|
+
this.target = target.name;
|
|
40
|
+
this.id = id;
|
|
41
|
+
this.workflowRunId = workflowRunId;
|
|
42
|
+
this.viewerUrl = sessionViewerUrl(id);
|
|
43
|
+
}
|
|
44
|
+
static async start(client, target, options) {
|
|
45
|
+
const deadline = Date.now() + target.startTimeout;
|
|
46
|
+
if (target.platform === 'web') {
|
|
47
|
+
const buildId = await resolveWebBuildId(client, target);
|
|
48
|
+
const sessionId = randomUUID();
|
|
49
|
+
const request = {
|
|
50
|
+
platform: 'web',
|
|
51
|
+
session_id: sessionId,
|
|
52
|
+
app_id: target.app.appId,
|
|
53
|
+
build_id: buildId,
|
|
54
|
+
idle_timeout_seconds: target.idleTimeoutSeconds,
|
|
55
|
+
web: {
|
|
56
|
+
...(target.viewport
|
|
57
|
+
? {
|
|
58
|
+
viewport: {
|
|
59
|
+
width: target.viewport.width,
|
|
60
|
+
height: target.viewport.height,
|
|
61
|
+
...(target.viewport.deviceScaleFactor ? { device_scale_factor: target.viewport.deviceScaleFactor } : {}),
|
|
62
|
+
},
|
|
63
|
+
}
|
|
64
|
+
: {}),
|
|
65
|
+
...(target.signIn
|
|
66
|
+
? { sign_in: { path: target.signIn.path, secret_name: target.signIn.secret, ...(target.signIn.persona ? { persona: target.signIn.persona } : {}) } }
|
|
67
|
+
: {}),
|
|
68
|
+
},
|
|
69
|
+
};
|
|
70
|
+
const response = await client.startDevice(request);
|
|
71
|
+
if (response.session_id !== sessionId) {
|
|
72
|
+
throw new VerifyError('SESSION_START_FAILED', `Revyl did not start the ${target.name} web session: ${response.error ?? response.message ?? 'the start was not acknowledged'}`);
|
|
73
|
+
}
|
|
74
|
+
const session = new DeviceSession(client, target, sessionId, response.workflow_run_id ?? '', options);
|
|
75
|
+
await session.untilReady(() => session.waitForWebReady(deadline));
|
|
76
|
+
return session;
|
|
77
|
+
}
|
|
78
|
+
const { app } = target;
|
|
79
|
+
const request = {
|
|
80
|
+
platform: target.platform,
|
|
81
|
+
is_simulation: true,
|
|
82
|
+
idle_timeout_seconds: target.idleTimeoutSeconds,
|
|
83
|
+
...(app.appId ? { app_id: app.appId } : {}),
|
|
84
|
+
...(app.buildId ? { build_id: app.buildId } : {}),
|
|
85
|
+
...(app.artifactUrl ? { app_url: app.artifactUrl } : {}),
|
|
86
|
+
...(target.device?.model ? { device_model: target.device.model } : {}),
|
|
87
|
+
...(target.device?.osVersion ? { os_version: target.device.osVersion } : {}),
|
|
88
|
+
};
|
|
89
|
+
const response = await client.startDevice(request);
|
|
90
|
+
if (!response.workflow_run_id) {
|
|
91
|
+
throw new VerifyError('SESSION_START_FAILED', `Revyl did not start the ${target.name} device: ${response.error ?? response.message ?? 'no workflow run was returned'}`);
|
|
92
|
+
}
|
|
93
|
+
const session = new DeviceSession(client, target, response.session_id ?? '', response.workflow_run_id, options);
|
|
94
|
+
await session.untilReady(async () => {
|
|
95
|
+
await session.waitForDeviceReady(deadline);
|
|
96
|
+
if (app.bundleId)
|
|
97
|
+
await session.launch(app.bundleId);
|
|
98
|
+
await session.waitForScreenContent(deadline);
|
|
99
|
+
});
|
|
100
|
+
return session;
|
|
101
|
+
}
|
|
102
|
+
get closed() {
|
|
103
|
+
return this.closePromise !== undefined;
|
|
104
|
+
}
|
|
105
|
+
/** Asks Revyl's agent to do something on the app, such as "add the first product to the cart". Throws INSTRUCTION_FAILED when it cannot. */
|
|
106
|
+
async instruction(text, options = {}) {
|
|
107
|
+
const result = await this.step('instruction', text, options.timeout);
|
|
108
|
+
if (!result.passed) {
|
|
109
|
+
throw new VerifyError('INSTRUCTION_FAILED', `Instruction failed: "${result.text}". ${sentence(result.reason)} See ${this.viewerUrl}`);
|
|
110
|
+
}
|
|
111
|
+
return result;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Asks Revyl's agent whether something holds on screen, such as "the cart shows 2 items". Throws VALIDATION_FAILED when it does not,
|
|
115
|
+
* unless soft is set, and STEP_FAILED when Revyl could not reach a verdict.
|
|
116
|
+
*/
|
|
117
|
+
async validation(text, options = {}) {
|
|
118
|
+
const result = await this.step('validation', text, options.timeout);
|
|
119
|
+
if (!result.passed && !options.soft) {
|
|
120
|
+
throw new VerifyError('VALIDATION_FAILED', `Validation failed: "${result.text}". ${sentence(result.reason)} See ${this.viewerUrl}`);
|
|
121
|
+
}
|
|
122
|
+
return result;
|
|
123
|
+
}
|
|
124
|
+
/** A PNG of the screen right now, also written to path when given. */
|
|
125
|
+
async screenshot(options = {}) {
|
|
126
|
+
const response = await this.worker('GET', 'screenshot', { retry: true });
|
|
127
|
+
if (!pngSize(response.body))
|
|
128
|
+
throw new VerifyError('ACTION_FAILED', 'Revyl returned a screenshot that is not a PNG image.');
|
|
129
|
+
if (options.path) {
|
|
130
|
+
const path = resolve(options.path);
|
|
131
|
+
await mkdir(dirname(path), { recursive: true });
|
|
132
|
+
await writeFile(path, response.body);
|
|
133
|
+
}
|
|
134
|
+
return response.body;
|
|
135
|
+
}
|
|
136
|
+
exec() {
|
|
137
|
+
return this.computerOnly('exec');
|
|
138
|
+
}
|
|
139
|
+
recording() {
|
|
140
|
+
return this.computerOnly('recording');
|
|
141
|
+
}
|
|
142
|
+
readFile() {
|
|
143
|
+
return this.computerOnly('readFile');
|
|
144
|
+
}
|
|
145
|
+
writeFile() {
|
|
146
|
+
return this.computerOnly('writeFile');
|
|
147
|
+
}
|
|
148
|
+
stat() {
|
|
149
|
+
return this.computerOnly('stat');
|
|
150
|
+
}
|
|
151
|
+
processes() {
|
|
152
|
+
return this.computerOnly('processes');
|
|
153
|
+
}
|
|
154
|
+
terminate() {
|
|
155
|
+
return this.computerOnly('terminate');
|
|
156
|
+
}
|
|
157
|
+
/** Stops the session on Revyl. Safe to call more than once. */
|
|
158
|
+
close() {
|
|
159
|
+
this.closePromise ??= (async () => {
|
|
160
|
+
try {
|
|
161
|
+
if (this.id)
|
|
162
|
+
await this.client.stopDeviceSession(this.id);
|
|
163
|
+
else
|
|
164
|
+
await this.client.cancelDevice(this.workflowRunId);
|
|
165
|
+
this.log('closed');
|
|
166
|
+
}
|
|
167
|
+
finally {
|
|
168
|
+
untrackSession(this);
|
|
169
|
+
}
|
|
170
|
+
})();
|
|
171
|
+
return this.closePromise;
|
|
172
|
+
}
|
|
173
|
+
[Symbol.asyncDispose]() {
|
|
174
|
+
return this.close();
|
|
175
|
+
}
|
|
176
|
+
async step(type, text, timeout) {
|
|
177
|
+
if (typeof text !== 'string' || text.trim() === '' || text.length > MAX_STEP_TEXT_LENGTH) {
|
|
178
|
+
throw new VerifyError('INVALID_ARGUMENT', `${type}() takes a description of 1 to ${MAX_STEP_TEXT_LENGTH} characters.`);
|
|
179
|
+
}
|
|
180
|
+
const timeoutMs = timeout ?? this.options.stepTimeout;
|
|
181
|
+
if (!Number.isInteger(timeoutMs) || timeoutMs < 1_000) {
|
|
182
|
+
throw new VerifyError('INVALID_ARGUMENT', `${type}() takes a timeout of at least 1000 ms.`);
|
|
183
|
+
}
|
|
184
|
+
if (this.stepRunning) {
|
|
185
|
+
throw new VerifyError('INVALID_ARGUMENT', 'Run one step at a time on a session: await each instruction or validation before starting the next.');
|
|
186
|
+
}
|
|
187
|
+
this.stepRunning = true;
|
|
188
|
+
const description = text.trim();
|
|
189
|
+
const startedAt = Date.now();
|
|
190
|
+
try {
|
|
191
|
+
const outcome = await this.runStep(type, description, timeoutMs);
|
|
192
|
+
const output = outcome.step_output ?? {};
|
|
193
|
+
const reason = output.status_reason ?? output.reasoning ?? undefined;
|
|
194
|
+
if (type === 'validation' && (outcome.success !== true || typeof output.validation_result !== 'boolean')) {
|
|
195
|
+
const why = outcome.success === true ? 'Revyl returned no verdict.' : sentence(reason);
|
|
196
|
+
throw new VerifyError('STEP_FAILED', `Validation did not reach a verdict: "${description}". ${why} See ${this.viewerUrl}`);
|
|
197
|
+
}
|
|
198
|
+
const passed = type === 'validation' ? output.validation_result === true : outcome.success === true && output.status !== 'failed';
|
|
199
|
+
const result = { type, text: description, passed, ...(reason ? { reason } : {}), durationMs: Date.now() - startedAt };
|
|
200
|
+
this.steps.push(result);
|
|
201
|
+
this.log(`${passed ? '✓' : '✗'} ${type} "${description}" (${formatDuration(result.durationMs)})${!passed && reason ? `: ${reason}` : ''}`);
|
|
202
|
+
return result;
|
|
203
|
+
}
|
|
204
|
+
catch (error) {
|
|
205
|
+
this.log(`✗ ${type} "${description}": ${isVerifyError(error) ? error.code : errorMessage(error)}`);
|
|
206
|
+
throw error;
|
|
207
|
+
}
|
|
208
|
+
finally {
|
|
209
|
+
this.stepRunning = false;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
async runStep(type, description, timeoutMs) {
|
|
213
|
+
const signal = AbortSignal.timeout(timeoutMs);
|
|
214
|
+
let stepId;
|
|
215
|
+
try {
|
|
216
|
+
const response = await this.worker('POST', 'execute_step', {
|
|
217
|
+
body: {
|
|
218
|
+
step_type: type,
|
|
219
|
+
step_description: description,
|
|
220
|
+
...(this.id ? { session_id: this.id } : {}),
|
|
221
|
+
workflow_run_id: this.workflowRunId,
|
|
222
|
+
},
|
|
223
|
+
signal,
|
|
224
|
+
timeoutMs,
|
|
225
|
+
});
|
|
226
|
+
const body = parseJson(response, 'execute_step');
|
|
227
|
+
if ('success' in body)
|
|
228
|
+
return body;
|
|
229
|
+
if (body.status !== 'accepted' || !body.step_id) {
|
|
230
|
+
throw new VerifyError('API_ERROR', 'Revyl answered execute_step without a result or an accepted step id.');
|
|
231
|
+
}
|
|
232
|
+
stepId = body.step_id;
|
|
233
|
+
for (let wait = STEP_POLL_BASE_MS;; wait = Math.min(wait * 2, STEP_POLL_MAX_MS)) {
|
|
234
|
+
await delay(wait, undefined, { signal });
|
|
235
|
+
const status = parseJson(await this.worker('GET', `step_status/${encodeURIComponent(stepId)}`, { signal, retry: true }), 'step_status');
|
|
236
|
+
if (status.status === 'running')
|
|
237
|
+
continue;
|
|
238
|
+
if (status.status === 'cancelled') {
|
|
239
|
+
throw new VerifyError('STEP_FAILED', `The ${type} was cancelled on Revyl before it finished: "${description}". See ${this.viewerUrl}`);
|
|
240
|
+
}
|
|
241
|
+
if (!STEP_TERMINAL_STATUSES.has(status.status) || !status.result) {
|
|
242
|
+
throw new VerifyError('API_ERROR', `Step ${stepId} reported status ${status.status} without a result Verify can read.`);
|
|
243
|
+
}
|
|
244
|
+
return status.result;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
catch (error) {
|
|
248
|
+
if (!signal.aborted)
|
|
249
|
+
throw error;
|
|
250
|
+
if (stepId) {
|
|
251
|
+
await this.client
|
|
252
|
+
.worker(this.workflowRunId, `step_cancel/${encodeURIComponent(stepId)}`, {
|
|
253
|
+
method: 'POST',
|
|
254
|
+
body: {},
|
|
255
|
+
signal: AbortSignal.timeout(STEP_CANCEL_TIMEOUT_MS),
|
|
256
|
+
})
|
|
257
|
+
.catch(() => undefined);
|
|
258
|
+
}
|
|
259
|
+
throw new VerifyError('TIMEOUT', `The ${type} did not finish within ${timeoutMs} ms and was cancelled: "${description}". See ${this.viewerUrl}`);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
async launch(bundleId) {
|
|
263
|
+
const response = await this.worker('POST', 'launch', { body: { bundle_id: bundleId } });
|
|
264
|
+
const result = parseJson(response, 'launch');
|
|
265
|
+
if (result.success !== true) {
|
|
266
|
+
throw new VerifyError('SESSION_START_FAILED', `Could not launch ${bundleId} on the ${this.target} device: ${result.error ?? 'no reason given'}`);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
async waitForScreenContent(startDeadline) {
|
|
270
|
+
const deadline = Math.min(startDeadline, Date.now() + 10_000);
|
|
271
|
+
while (Date.now() < deadline) {
|
|
272
|
+
try {
|
|
273
|
+
const response = await this.client.worker(this.workflowRunId, 'screenshot', {
|
|
274
|
+
method: 'GET', retry: false, signal: AbortSignal.timeout(Math.max(1, deadline - Date.now())),
|
|
275
|
+
});
|
|
276
|
+
if (response.status === 200) {
|
|
277
|
+
const content = screenHasContent(response.body);
|
|
278
|
+
if (content === true)
|
|
279
|
+
return;
|
|
280
|
+
if (content === undefined)
|
|
281
|
+
break;
|
|
282
|
+
}
|
|
283
|
+
else if (response.status !== 409 && response.status !== 503) {
|
|
284
|
+
break;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
catch {
|
|
288
|
+
break;
|
|
289
|
+
}
|
|
290
|
+
const remaining = deadline - Date.now();
|
|
291
|
+
if (remaining > 0)
|
|
292
|
+
await delay(Math.min(500, remaining));
|
|
293
|
+
}
|
|
294
|
+
this.log('debug: best-effort first-screen wait ended without non-blank content; the runtime has no rendered-readiness signal');
|
|
295
|
+
}
|
|
296
|
+
log(line) {
|
|
297
|
+
this.options.log?.(`${this.target}: ${line}`);
|
|
298
|
+
}
|
|
299
|
+
computerOnly(call) {
|
|
300
|
+
return Promise.reject(new VerifyError('NOT_AVAILABLE_ON_PLATFORM', `${call}() runs on computer targets (platform 'mac' or 'linux'), not on ${this.platform} sessions.`));
|
|
301
|
+
}
|
|
302
|
+
async worker(method, action, options) {
|
|
303
|
+
if (this.closePromise)
|
|
304
|
+
throw new VerifyError('SESSION_ENDED', `The ${this.target} session was closed.`);
|
|
305
|
+
const response = await this.client.worker(this.workflowRunId, action, { method, ...options });
|
|
306
|
+
if (response.status >= 200 && response.status < 300)
|
|
307
|
+
return response;
|
|
308
|
+
throw this.workerError(method, action, response);
|
|
309
|
+
}
|
|
310
|
+
workerError(method, action, response) {
|
|
311
|
+
const baseAction = action.split('/')[0] ?? action;
|
|
312
|
+
const code = responseErrorCode(response);
|
|
313
|
+
const detail = responseDetail(response);
|
|
314
|
+
const reason = detail ?? code ?? `HTTP ${response.status}`;
|
|
315
|
+
if (response.status === 401 || response.status === 403)
|
|
316
|
+
return apiError(method, `device-proxy/${baseAction}`, response);
|
|
317
|
+
if (response.status === 501 || code === 'not_available_on_platform' || code === 'step_output_not_available') {
|
|
318
|
+
if (baseAction === 'execute_step' || baseAction === 'step_status') {
|
|
319
|
+
return new VerifyError('NOT_AVAILABLE_ON_PLATFORM', `The Revyl backend at ${this.client.backendUrl} does not run instruction and validation steps on ${this.platform} sessions (${code ?? 'HTTP 501'}).`);
|
|
320
|
+
}
|
|
321
|
+
return new VerifyError('NOT_AVAILABLE_ON_PLATFORM', `${baseAction} is not available on ${this.platform} sessions (${reason}).`);
|
|
322
|
+
}
|
|
323
|
+
if (response.status === 404 || response.status === 410) {
|
|
324
|
+
return new VerifyError('SESSION_ENDED', `The ${this.target} session is no longer reachable (${reason}). See ${this.viewerUrl}`);
|
|
325
|
+
}
|
|
326
|
+
if (response.status === 409) {
|
|
327
|
+
if (SESSION_NOT_RUNNING.test(reason)) {
|
|
328
|
+
return new VerifyError('SESSION_ENDED', `The ${this.target} session is no longer running (${reason}). See ${this.viewerUrl}`);
|
|
329
|
+
}
|
|
330
|
+
return new VerifyError('SESSION_BUSY', `The ${this.target} session is busy and refused ${baseAction}: ${reason}`);
|
|
331
|
+
}
|
|
332
|
+
if (method === 'POST' && (isExecutionUncertain(response) || (code && UNCERTAIN_ERROR_CODES.has(code)))) {
|
|
333
|
+
return new VerifyError('ACTION_UNCERTAIN', `${baseAction} may or may not have been applied (${reason}); Verify does not repeat it. Check the session at ${this.viewerUrl}`);
|
|
334
|
+
}
|
|
335
|
+
return new VerifyError('ACTION_FAILED', `${baseAction} failed on the ${this.target} session (HTTP ${response.status}): ${reason}`);
|
|
336
|
+
}
|
|
337
|
+
async untilReady(waitForReady) {
|
|
338
|
+
trackSession(this);
|
|
339
|
+
this.log(`starting ${this.platform} session ${this.viewerUrl}`);
|
|
340
|
+
try {
|
|
341
|
+
await waitForReady();
|
|
342
|
+
}
|
|
343
|
+
catch (error) {
|
|
344
|
+
await this.close().catch(() => undefined);
|
|
345
|
+
throw error;
|
|
346
|
+
}
|
|
347
|
+
this.log('ready');
|
|
348
|
+
}
|
|
349
|
+
async waitForWebReady(deadline) {
|
|
350
|
+
for (;;) {
|
|
351
|
+
const detail = await this.client.getDeviceSession(this.id);
|
|
352
|
+
if (ENDED_STATUSES.has(detail.status)) {
|
|
353
|
+
throw new VerifyError('SESSION_START_FAILED', `The ${this.target} web session ended before it was ready (${detail.status}): ${detail.error_message ?? 'no reason given'}. See ${this.viewerUrl}`);
|
|
354
|
+
}
|
|
355
|
+
if (detail.status === 'running' && detail.workflow_run_id) {
|
|
356
|
+
this.workflowRunId = detail.workflow_run_id;
|
|
357
|
+
const response = await this.client.worker(this.workflowRunId, 'screenshot', { method: 'GET' });
|
|
358
|
+
if (response.status === 200 && pngSize(response.body))
|
|
359
|
+
return;
|
|
360
|
+
if (response.status !== 503 && response.status !== 409 && responseErrorCode(response) !== 'not_ready') {
|
|
361
|
+
throw this.workerError('GET', 'screenshot', response);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
await waitBeforeDeadline(deadline, `The ${this.target} web session was not ready in time. See ${this.viewerUrl}`);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
async waitForDeviceReady(deadline) {
|
|
368
|
+
for (;;) {
|
|
369
|
+
const connection = await this.client.getWorkerConnection(this.workflowRunId);
|
|
370
|
+
if (connection.status === 'ready' && connection.worker_ws_url)
|
|
371
|
+
break;
|
|
372
|
+
if (connection.status === 'cancelled' || connection.status === 'stopped' || connection.status === 'failed') {
|
|
373
|
+
throw new VerifyError('SESSION_START_FAILED', `The ${this.target} device did not start (${connection.status}): ${connection.message ?? 'no reason given'}`);
|
|
374
|
+
}
|
|
375
|
+
await waitBeforeDeadline(deadline, `The ${this.target} device was not assigned in time.`);
|
|
376
|
+
}
|
|
377
|
+
const connectDeadline = Math.min(deadline, Date.now() + DEVICE_CONNECT_TIMEOUT_MS);
|
|
378
|
+
for (;;) {
|
|
379
|
+
const response = await this.client.worker(this.workflowRunId, 'health', { method: 'GET', retry: true });
|
|
380
|
+
if (response.status === 200 && parseJson(response, 'health').device_connected)
|
|
381
|
+
return;
|
|
382
|
+
if (response.status !== 200 && response.status !== 503)
|
|
383
|
+
throw this.workerError('GET', 'health', response);
|
|
384
|
+
await waitBeforeDeadline(connectDeadline, `The ${this.target} device did not connect in time.`);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
async function resolveWebBuildId(client, target) {
|
|
389
|
+
const { app } = target;
|
|
390
|
+
if (app.buildId)
|
|
391
|
+
return app.buildId;
|
|
392
|
+
if (app.url) {
|
|
393
|
+
const build = await client.registerWebUrlBuild(app.appId, { url: app.url, ...(app.branch ? { branch_name: app.branch } : {}) });
|
|
394
|
+
if (!build.id)
|
|
395
|
+
throw new VerifyError('API_ERROR', `Revyl registered ${app.url} without returning a build id.`);
|
|
396
|
+
return build.id;
|
|
397
|
+
}
|
|
398
|
+
let branchId;
|
|
399
|
+
if (!app.branch) {
|
|
400
|
+
const detail = await client.getApp(app.appId);
|
|
401
|
+
if (detail.platform && detail.platform.toLowerCase() !== 'web') {
|
|
402
|
+
throw new VerifyError('CONFIG_INVALID', `Target ${target.name}: Revyl app ${app.appId} is a ${detail.platform} app, not a web app.`);
|
|
403
|
+
}
|
|
404
|
+
branchId = detail.default_branch_id ?? undefined;
|
|
405
|
+
}
|
|
406
|
+
const latest = await client.latestWebBuild(app.appId, { branchId, branchName: app.branch });
|
|
407
|
+
if (!latest?.id) {
|
|
408
|
+
throw new VerifyError('SESSION_START_FAILED', `Target ${target.name}: Revyl app ${app.appId} has no web build${app.branch ? ` on branch ${app.branch}` : ''}. ` +
|
|
409
|
+
"Set app.url to a deployed HTTPS URL, or build one with 'revyl build --platform web'.");
|
|
410
|
+
}
|
|
411
|
+
return latest.id;
|
|
412
|
+
}
|
|
413
|
+
async function waitBeforeDeadline(deadline, timeoutMessage) {
|
|
414
|
+
const remaining = deadline - Date.now();
|
|
415
|
+
if (remaining <= 0)
|
|
416
|
+
throw new VerifyError('TIMEOUT', timeoutMessage);
|
|
417
|
+
await delay(Math.min(POLL_INTERVAL_MS, remaining));
|
|
418
|
+
}
|
|
419
|
+
function isExecutionUncertain(response) {
|
|
420
|
+
try {
|
|
421
|
+
return JSON.parse(new TextDecoder().decode(response.body)).execution_uncertain === true;
|
|
422
|
+
}
|
|
423
|
+
catch {
|
|
424
|
+
return false;
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
|
|
428
|
+
export function pngSize(bytes) {
|
|
429
|
+
if (bytes.length < 24 || PNG_SIGNATURE.some((value, index) => bytes[index] !== value))
|
|
430
|
+
return undefined;
|
|
431
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
432
|
+
const width = view.getUint32(16);
|
|
433
|
+
const height = view.getUint32(20);
|
|
434
|
+
return width > 0 && height > 0 ? { width, height } : undefined;
|
|
435
|
+
}
|
|
436
|
+
function formatDuration(ms) {
|
|
437
|
+
return ms < 60_000 ? `${(ms / 1_000).toFixed(1)}s` : `${Math.floor(ms / 60_000)}m ${Math.round((ms % 60_000) / 1_000)}s`;
|
|
438
|
+
}
|
|
439
|
+
function sentence(reason) {
|
|
440
|
+
if (!reason)
|
|
441
|
+
return 'Revyl gave no reason.';
|
|
442
|
+
return /[.!?]$/.test(reason.trim()) ? reason.trim() : `${reason.trim()}.`;
|
|
443
|
+
}
|
package/dist/version.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { fileURLToPath } from 'node:url';
|
|
3
|
+
const manifestPath = fileURLToPath(new URL('../package.json', import.meta.url));
|
|
4
|
+
const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
|
|
5
|
+
export const VERSION = manifest.version;
|
|
6
|
+
export const PACKAGE_NAME = manifest.name;
|
package/docs/README.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Verify documentation
|
|
2
|
+
|
|
3
|
+
These files ship inside the npm package at `node_modules/@revyl/verify/docs/`, so a coding agent working in your project can read the docs for the exact version you installed.
|
|
4
|
+
|
|
5
|
+
| File | Read it when you want to |
|
|
6
|
+
| --- | --- |
|
|
7
|
+
| [end-to-end.md](end-to-end.md) | Review live verification scope, observed outcomes, and evidence limits |
|
|
8
|
+
| [getting-started.md](getting-started.md) | Install Verify, authenticate, and run a first script |
|
|
9
|
+
| [api.md](api.md) | Look up session lifecycle, agent steps, screenshots, commands, files, and processes |
|
|
10
|
+
| [configuration.md](configuration.md) | Look up every option in `verify.config.ts`, the environment variables, and the CLI |
|
|
11
|
+
| [web.md](web.md) | Open a web app in a Revyl cloud browser |
|
|
12
|
+
| [mobile.md](mobile.md) | Open an iOS or Android app on a Revyl cloud device |
|
|
13
|
+
| [computers.md](computers.md) | Open a Mac or Linux computer for commands, files, processes, screenshots, and optional recording metadata |
|
|
14
|
+
| [linux-computers-contract.md](linux-computers-contract.md) | Read the implemented shared Linux computer contract |
|
|
15
|
+
| [ci.md](ci.md) | Call Verify from CI or from a test runner you already use |
|
|
16
|
+
| [errors.md](errors.md) | Understand an error code and fix it |
|
|
17
|
+
| [agents.md](agents.md) | Use Verify as a coding agent |
|
|
18
|
+
|
|
19
|
+
[llms.txt](llms.txt) lists the same files in the llms.txt format.
|
package/docs/agents.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# For coding agents
|
|
2
|
+
|
|
3
|
+
This page is for AI coding agents (Claude Code, Cursor, Codex, and others) that use Verify to check an app on a cloud browser/device or run commands and files on a Mac or Linux computer. The rest of this folder describes the API in full; this page is the workflow.
|
|
4
|
+
|
|
5
|
+
## Workflow
|
|
6
|
+
|
|
7
|
+
1. Read `verify.config.ts` to learn the target names and their platforms. If there is none, run `npx @revyl/verify init --platform <web|ios|android> --app-id <id>`, or `npx @revyl/verify init --platform mac` without an app id. You can also pass a target object to `open()`.
|
|
8
|
+
2. Check the credentials once with `npx @revyl/verify whoami`. It starts no session.
|
|
9
|
+
3. Write a short script, or extend an existing one, that opens one target, runs the steps you need, and closes:
|
|
10
|
+
```ts
|
|
11
|
+
import { open } from '@revyl/verify';
|
|
12
|
+
|
|
13
|
+
const session = await open('ios');
|
|
14
|
+
try {
|
|
15
|
+
await session.instruction('Sign in as the demo user');
|
|
16
|
+
await session.validation('The dashboard shows at least one project card');
|
|
17
|
+
await session.screenshot({ path: '.verify/dashboard.png' });
|
|
18
|
+
} finally {
|
|
19
|
+
await session.close();
|
|
20
|
+
}
|
|
21
|
+
```
|
|
22
|
+
4. Run it with `node path/to/script.ts`. Share the session link from the first `[verify]` line with the user, so they can watch.
|
|
23
|
+
5. Read the outcome from the exit code and stderr. Each step prints `✓` or `✗` with its text; a failure prints `CODE: message`, ending with the session link. Look at any screenshot you saved before changing the script or the app.
|
|
24
|
+
|
|
25
|
+
## Rules
|
|
26
|
+
|
|
27
|
+
Mac and Linux computers are early access and not yet available in Revyl production. They work only on Revyl staging for organizations enabled for the requested OS through the optional `REVYL_BACKEND_URL` override with matching staging credentials. Read [availability and errors](computers.md#availability-and-errors) before starting a computer; do not try production or repeat an uncertain start/stop to probe availability.
|
|
28
|
+
|
|
29
|
+
For computer work, use structured `exec(['program', 'arg'])`, `readFile`, `writeFile`, `stat`, and run-owned handles from `processes()`/`terminate(id)`. Check command output with ordinary assertions and close in `finally`; screenshots are supported, but instruction/validation are not. Do not fall back to agent steps on computers or shell/file methods on mobile/web. See [computers.md](computers.md) for limits and staging setup. Linux uses inline targets or config entries; Linux init is intentionally unavailable. Starting a computer is a paid external action and needs the same approval as starting a device.
|
|
30
|
+
|
|
31
|
+
Computers have a hard 60-minute lifetime from create, not from readiness or activity. `expiresAt` is optional on older APIs; expiry does not prevent close. After terminal state and confirmed release, `recording()` can return private signed metadata or null, including after close. Never log or share its signed URL/query. Do not automatically wait for recordings or retry `COMPUTER_LIMIT` starts: queued work (including iOS) or the concurrency limit may be responsible.
|
|
32
|
+
|
|
33
|
+
- Write steps the way you would brief a person: what to do, or what should be true, in one sentence. Name visible things ("the blue Save button in the footer"), not implementation details.
|
|
34
|
+
- Use `instruction` for doing and `validation` for checking. Do not ask an instruction to check something, or a validation to change anything.
|
|
35
|
+
- Prefer several small steps to one long one. Small steps fail with clearer reasons and time out less.
|
|
36
|
+
- Each step stands alone: Revyl does not carry values from one step to the next. Put everything a step needs into its text.
|
|
37
|
+
- Every step runs Revyl's agent again; nothing is cached or replayed. Do not rerun a whole script to probe one step.
|
|
38
|
+
- Await each step. Two steps at once on one session throw `INVALID_ARGUMENT`.
|
|
39
|
+
- Do not catch `VerifyError` to make a check pass. A `NOT_AVAILABLE_ON_PLATFORM` error means the call is not offered on that platform; see [web.md](web.md) and [computers.md](computers.md). A `STEP_FAILED` error means the step did not run to an outcome, not that the app is wrong.
|
|
40
|
+
- Never automatically repeat a computer mutation after `ACTION_UNCERTAIN` or a timeout: the command may have run, and timeout does not prove termination or rollback. Do not log argv, stdin, file contents, or credentials.
|
|
41
|
+
- Always close what you open in `try/finally`. Each open session is paid. Automatic cleanup is best effort with a 15-second bound and skips signal/error events that already have listeners. Device close acknowledges stop, while computer close requires confirmed terminal release.
|
|
42
|
+
- Never put secrets in scripts or config. Credentials come from `REVYL_API_KEY` or a Revyl CLI login; web sign-in uses the **name** of a Revyl organization secret.
|
|
43
|
+
- Step descriptions are sent to Revyl and logged to stderr by default. Do not include private credentials or customer data in instruction or validation text; use test-only sign-in and synthetic data instead. `log: false` suppresses local progress, not remote processing or reports.
|
package/docs/api.md
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# API
|
|
2
|
+
|
|
3
|
+
```ts
|
|
4
|
+
import { open, defineConfig, VerifyError, isVerifyError } from '@revyl/verify';
|
|
5
|
+
```
|
|
6
|
+
|
|
7
|
+
## open(target, options?)
|
|
8
|
+
|
|
9
|
+
Linux targets are supported **since 0.2.0**; 0.1.0 includes the target type but rejects Linux locally.
|
|
10
|
+
|
|
11
|
+
Starts a Revyl cloud browser, device, or Mac or Linux computer. Mobile first-screen readiness is [best-effort](mobile.md#steps), not a foreground/rendered-readiness guarantee. Computer starts retain a 10-minute default timeout; an unclaimed start may fail after 120 seconds with no-host context.
|
|
12
|
+
|
|
13
|
+
Both computer operating systems are early access on staging only for organizations enabled for the requested OS, not yet available in Revyl production. Use the optional `REVYL_BACKEND_URL` override and staging credentials described in [computer availability](computers.md#availability-and-errors).
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
const web = await open('web'); // a target named in verify.config.ts
|
|
17
|
+
const ios = await open({ platform: 'ios', app: { appId: '…' } }); // or a target object, no config needed
|
|
18
|
+
const mac = await open({ platform: 'mac' }); // computer, no app needed
|
|
19
|
+
const linux = await open({ platform: 'linux' }); // same computer surface
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
- `target`: a target name from `verify.config.ts`, or a target object with the same shape as a config entry (see [configuration.md](configuration.md)). Verify reads the nearest `verify.config.ts` at or above the current folder.
|
|
23
|
+
- `options.config`: the path of a config file to read instead.
|
|
24
|
+
- `options.log`: where progress lines go. By default Verify writes the session link and each step's outcome to stderr, prefixed with `[verify]`. Pass a function to receive the lines yourself, or `false` to turn them off.
|
|
25
|
+
|
|
26
|
+
`open` needs credentials: `REVYL_API_KEY` or a Revyl CLI login. It throws `AUTH_MISSING` without them, `CONFIG_NOT_FOUND` or `CONFIG_INVALID` for config problems, and `SESSION_START_FAILED` or `TIMEOUT` when the browser or device does not start.
|
|
27
|
+
|
|
28
|
+
## session.instruction(text, options?)
|
|
29
|
+
|
|
30
|
+
Available on web/iOS/Android only. Computers throw `NOT_AVAILABLE_ON_PLATFORM` without sending an agent request.
|
|
31
|
+
|
|
32
|
+
Asks Revyl's agent to do something in the app, described the way you would tell a person: `'Sign in as the demo user and open Settings'`. The agent looks at the screen and acts until it is done or decides it cannot.
|
|
33
|
+
|
|
34
|
+
- Resolves with a `StepResult` when the agent finishes.
|
|
35
|
+
- Throws `INSTRUCTION_FAILED`, with Revyl's reason and the session link, when it cannot, and `STEP_FAILED` when the step is cancelled on Revyl.
|
|
36
|
+
- `options.timeout`: milliseconds before the step is cancelled on Revyl (`TIMEOUT`). Defaults to `stepTimeout` in the config, or 180000.
|
|
37
|
+
|
|
38
|
+
## session.validation(text, options?)
|
|
39
|
+
|
|
40
|
+
Available on web/iOS/Android only, not computers.
|
|
41
|
+
|
|
42
|
+
Asks Revyl's agent whether something holds on screen right now: `'The cart shows 2 items and a total of $40'`.
|
|
43
|
+
|
|
44
|
+
- Resolves with a `StepResult` whose `passed` is `true` when it holds.
|
|
45
|
+
- Throws `VALIDATION_FAILED`, with Revyl's reason and the session link, when it does not. This is what makes a plain script exit non-zero.
|
|
46
|
+
- Throws `STEP_FAILED` when Revyl cannot reach a verdict, for example because the step errored or was cancelled on Revyl.
|
|
47
|
+
- `options.soft: true` returns a failed result instead of throwing `VALIDATION_FAILED`, for scripts that want to collect several outcomes. It does not turn `STEP_FAILED` into a result.
|
|
48
|
+
- `options.timeout`: as for `instruction`.
|
|
49
|
+
|
|
50
|
+
## StepResult
|
|
51
|
+
|
|
52
|
+
```ts
|
|
53
|
+
interface StepResult {
|
|
54
|
+
type: 'instruction' | 'validation';
|
|
55
|
+
text: string; // the step as sent, trimmed
|
|
56
|
+
passed: boolean;
|
|
57
|
+
reason?: string; // Revyl's explanation, when it gave one
|
|
58
|
+
durationMs: number;
|
|
59
|
+
}
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`session.steps` lists every step run on the session, in order.
|
|
63
|
+
|
|
64
|
+
When inspecting `revyl device report --session-id <id>`, read validation verdicts from the step's `effective_status` and `validation_result`, with `validations_passed` / `total_validations` for the validation summary. `passed_steps` / `failed_steps` and transport `status` are not enough: a successfully executed validation can have a false verdict. The SDK's `StepResult.passed` reflects that verdict, not just transport success.
|
|
65
|
+
|
|
66
|
+
## session.screenshot(options?)
|
|
67
|
+
|
|
68
|
+
Resolves with a PNG of the screen as a `Uint8Array`. With `options.path`, also writes it there, creating folders as needed.
|
|
69
|
+
|
|
70
|
+
## session.close()
|
|
71
|
+
|
|
72
|
+
Requests that Revyl stop the session. For web/mobile, success acknowledges the stop request rather than proving physical release. It is safe to call more than once, and steps after it throw `SESSION_ENDED`. A `Session` also works with `await using`, which closes it at the end of the block.
|
|
73
|
+
|
|
74
|
+
For computers, cleanup must be confirmed; a terminal state alone does not prove physical release. Uncertain release throws `ACTION_UNCERTAIN`.
|
|
75
|
+
|
|
76
|
+
## Computer commands, files, and processes
|
|
77
|
+
|
|
78
|
+
Mac and Linux sessions provide `exec(argv: string[], options?)`, `readFile(path, options?)` returning `Buffer`, `writeFile(path, string | Uint8Array, options?)`, `stat(path, options?)` returning `{ kind, size }`, `processes(options?)` returning `{ id, state, exitCode? }[]`, and `terminate(id, options?)`. They throw `NOT_AVAILABLE_ON_PLATFORM` on web/iOS/Android. See [computers.md](computers.md) for complete signatures, limits, timeouts, and examples. Both support `screenshot()`; instruction/validation do not run on computers.
|
|
79
|
+
|
|
80
|
+
### `session.recording(): Promise<ComputerRecording | null>`
|
|
81
|
+
|
|
82
|
+
**Since 0.2.0.** This method is not available in 0.1.0.
|
|
83
|
+
|
|
84
|
+
Computers only. First reads the report and requires terminal state plus confirmed release, otherwise throws `SESSION_BUSY`. Returns `{ url: string, durationSeconds?: number, startedAt?: Date }` or null for an unavailable recording. Nullable optional wire fields are omitted. This method works after `close()`, does not wait for upload, and does not download or fetch the URL. Keep the signed HTTPS URL and its query private. Request fresh metadata when the URL expires. The normal report has no recording URL. Devices reject this call locally with `NOT_AVAILABLE_ON_PLATFORM`.
|
|
85
|
+
|
|
86
|
+
## Other session properties
|
|
87
|
+
|
|
88
|
+
| Property | Meaning |
|
|
89
|
+
| --- | --- |
|
|
90
|
+
| `viewerUrl` | The session in Revyl: a live view while it runs, its recording/report after. Use `revyl device report --session-id <id>` for web/mobile, or `revyl computer report <id>` for computers. |
|
|
91
|
+
| `id` | Revyl's session id |
|
|
92
|
+
| `platform` | `'web'`, `'ios'`, `'android'`, `'mac'`, or `'linux'` |
|
|
93
|
+
| `expiresAt` | **Since 0.2.0.** Validated API expiry as a `Date`, or undefined for devices and older APIs. Computers have a hard 60-minute lifetime from create. Operations/screenshot observation are bounded by expiry; close and recording remain possible. |
|
|
94
|
+
| `target` | The target name, or the platform for a target object |
|
|
95
|
+
| `closed` | Whether `close()` was called |
|
|
96
|
+
|
|
97
|
+
## Rules worth knowing
|
|
98
|
+
|
|
99
|
+
- **One step at a time.** Await each step before starting the next on the same session; a second concurrent step throws `INVALID_ARGUMENT`. Separate sessions can run in parallel.
|
|
100
|
+
- **No hidden retries.** Verify never repeats a step or a device action. A step whose outcome is unknown because the connection failed mid-request throws `ACTION_UNCERTAIN`.
|
|
101
|
+
- **No memory between steps.** Each step is independent: Revyl does not carry values from one step to the next. Put everything a step needs into its text.
|
|
102
|
+
- **No replay or cache.** Every step runs Revyl's agent again, every time.
|
|
103
|
+
|
|
104
|
+
## Cleanup
|
|
105
|
+
|
|
106
|
+
While a session is open, Verify attempts best-effort cleanup with a 15-second budget:
|
|
107
|
+
|
|
108
|
+
- on Ctrl+C or SIGTERM: it attempts to close open sessions, then exits with 130 or 143 (a second Ctrl+C exits at once);
|
|
109
|
+
- on an uncaught error, such as a failed validation that nothing catches: it prints the error, attempts cleanup, then exits with 1;
|
|
110
|
+
- when the script finishes without calling `close()`: it attempts cleanup and prints a reminder.
|
|
111
|
+
|
|
112
|
+
Verify skips installing a signal or error handler when that event already has listeners, so test runners and apps that manage their own shutdown keep control of it. Always close in `try/finally` or an `after` hook; automatic cleanup cannot guarantee success. A process that is killed outright, or calls `process.exit()` with sessions open, leaves browser/mobile sessions running until their idle timeout (10 minutes by default). Computers have a server-bounded one-hour lifetime instead; no idle timeout is configurable.
|
|
113
|
+
|
|
114
|
+
## defineConfig(config)
|
|
115
|
+
|
|
116
|
+
Returns the config unchanged; it exists for type checking in `verify.config.ts`. See [configuration.md](configuration.md).
|
|
117
|
+
|
|
118
|
+
## Errors
|
|
119
|
+
|
|
120
|
+
Everything Verify throws on purpose is a `VerifyError` with a stable `code`; `isVerifyError(error)` narrows to it. See [errors.md](errors.md).
|