@somewhere-tech/cli 0.34.3 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/browser.js.map +1 -1
- package/dist/commands/init.js +135 -16
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/status.js +2 -13
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/verify.js +526 -4
- package/dist/commands/verify.js.map +1 -1
- package/dist/lib/browser-actions.js +11 -1
- package/dist/lib/browser-actions.js.map +1 -1
- package/dist/lib/browser-run.js +14 -0
- package/dist/lib/browser-run.js.map +1 -1
- package/dist/lib/declared-data.js +41 -51
- package/dist/lib/declared-data.js.map +1 -1
- package/dist/lib/files.js +3 -2
- package/dist/lib/files.js.map +1 -1
- package/dist/lib/init-agent-guide.js +51 -29
- package/dist/lib/init-agent-guide.js.map +1 -1
- package/dist/lib/init-feature-template.js +1397 -0
- package/dist/lib/init-feature-template.js.map +1 -0
- package/dist/lib/init-features.js +120 -0
- package/dist/lib/init-features.js.map +1 -0
- package/dist/lib/init-green-template.js +1 -1
- package/dist/lib/init-scaffold.js +13 -5
- package/dist/lib/init-scaffold.js.map +1 -1
- package/dist/lib/init-template.js +7 -7
- package/dist/lib/typecheck.js +7 -2
- package/dist/lib/typecheck.js.map +1 -1
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/runtime/DECLARED-DATA-VENDOR.json +7 -5
- package/runtime/declared-data.cjs +722 -98
- package/dist/lib/init-auth-template.js +0 -230
- package/dist/lib/init-auth-template.js.map +0 -1
package/dist/commands/verify.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { randomBytes } from 'node:crypto';
|
|
1
2
|
import { existsSync, readFileSync } from 'node:fs';
|
|
2
3
|
import { dirname, resolve } from 'node:path';
|
|
3
4
|
import { ApiClient, CliApiError } from '../lib/client.js';
|
|
@@ -137,6 +138,9 @@ export function normalizeVerifyFlow(raw, baseDir = process.cwd()) {
|
|
|
137
138
|
}
|
|
138
139
|
if (!isRecord(raw))
|
|
139
140
|
throw new Error('flow must be a JSON object.');
|
|
141
|
+
if (raw.actors !== undefined || raw.journey !== undefined) {
|
|
142
|
+
throw new Error('This is a multi-user flow (actors + journey). Run it with `somewhere verify --project <project> --flow <file>`.');
|
|
143
|
+
}
|
|
140
144
|
const supported = new Set(['actions', 'auth', 'local_storage', 'cookies', 'headers', 'expect_requests', 'visible_only', 'viewports']);
|
|
141
145
|
const unknown = Object.keys(raw).filter((key) => !supported.has(key));
|
|
142
146
|
if (unknown.length)
|
|
@@ -203,6 +207,8 @@ function actionName(action, fallback) {
|
|
|
203
207
|
return `expect ${action.expect.selector}`;
|
|
204
208
|
if ('screenshot' in action)
|
|
205
209
|
return `screenshot ${action.screenshot}`;
|
|
210
|
+
if ('goto' in action)
|
|
211
|
+
return `goto ${action.goto}`;
|
|
206
212
|
return `eval ${action.eval}`;
|
|
207
213
|
}
|
|
208
214
|
function reportPassed(report) {
|
|
@@ -225,19 +231,24 @@ function shapeVerificationReport(target, flow, runs) {
|
|
|
225
231
|
const failedRequests = [];
|
|
226
232
|
const expectations = [];
|
|
227
233
|
const screenshots = [];
|
|
228
|
-
|
|
234
|
+
const layout = [];
|
|
235
|
+
for (const { viewport, report, actions: runActions } of runs) {
|
|
236
|
+
const actions = runActions ?? flow.actions;
|
|
237
|
+
if (typeof report.accessibility_layout === 'string' && report.accessibility_layout) {
|
|
238
|
+
layout.push({ viewport: viewport.label, detail: report.accessibility_layout });
|
|
239
|
+
}
|
|
229
240
|
for (const [index, step] of (report.steps ?? []).entries()) {
|
|
230
241
|
const sourceIndex = typeof step.step === 'number' ? step.step : index;
|
|
231
242
|
// The local browser models its requested final capture as an internal
|
|
232
243
|
// screenshot step. Hosted capture_after does not, so keep the public
|
|
233
244
|
// verification steps limited to the caller's flow on both halves.
|
|
234
|
-
if (sourceIndex >=
|
|
245
|
+
if (sourceIndex >= actions.length && step.action === 'screenshot')
|
|
235
246
|
continue;
|
|
236
247
|
const passed = (step.ok ?? step.passed) !== false;
|
|
237
248
|
steps.push({
|
|
238
249
|
viewport: viewport.label,
|
|
239
250
|
step: sourceIndex + 1,
|
|
240
|
-
name: actionName(
|
|
251
|
+
name: actionName(actions[sourceIndex], step.action ?? 'page load'),
|
|
241
252
|
passed,
|
|
242
253
|
...(step.error ? { error: step.error } : {}),
|
|
243
254
|
...(typeof step.duration_ms === 'number' ? { duration_ms: step.duration_ms } : {}),
|
|
@@ -315,6 +326,7 @@ function shapeVerificationReport(target, flow, runs) {
|
|
|
315
326
|
},
|
|
316
327
|
},
|
|
317
328
|
screenshots,
|
|
329
|
+
layout,
|
|
318
330
|
};
|
|
319
331
|
}
|
|
320
332
|
export async function runVerification(target, flow, client) {
|
|
@@ -371,6 +383,434 @@ export async function runVerification(target, flow, client) {
|
|
|
371
383
|
}));
|
|
372
384
|
return shapeVerificationReport(targetLabel, flow, runs);
|
|
373
385
|
}
|
|
386
|
+
// ── Multi-user journeys (tsk_f49ccbb75f994f1fb5179198087d24de) ──────────────
|
|
387
|
+
//
|
|
388
|
+
// Several signed-in people in one run: each actor is its own named browser
|
|
389
|
+
// session on the platform (a separate browser, so no cookie, storage or cache
|
|
390
|
+
// is shared between actors), and the journey's segments run in the order
|
|
391
|
+
// written, each as one ordinary bounded browser call. Every call carries the
|
|
392
|
+
// project, so the platform's owned-origin rules apply to every actor on every
|
|
393
|
+
// step. Sessions this run opened are closed when it ends, whatever the outcome.
|
|
394
|
+
export const VERIFY_JOURNEY_LIMITS = {
|
|
395
|
+
actors: 3,
|
|
396
|
+
segments: 12,
|
|
397
|
+
actions_per_segment: 30,
|
|
398
|
+
total_actions: 120,
|
|
399
|
+
deadline_ms: 10 * 60 * 1000,
|
|
400
|
+
segment_timeout_ms: VERIFY_TIMEOUT_MS,
|
|
401
|
+
/** Separate from the journey budget: how long closing the actors' browsers
|
|
402
|
+
* may take once the run stops, including waiting for a call in flight. */
|
|
403
|
+
cleanup_budget_ms: 20_000,
|
|
404
|
+
};
|
|
405
|
+
const ACTOR_NAME = /^[a-z][a-z0-9_-]{0,15}$/;
|
|
406
|
+
function isJourneyPath(value) {
|
|
407
|
+
return typeof value === 'string' && value.startsWith('/') && !value.startsWith('//') && !/[\\\s]/.test(value);
|
|
408
|
+
}
|
|
409
|
+
export function isVerifyJourneyInput(raw) {
|
|
410
|
+
return isRecord(raw) && (raw.actors !== undefined || raw.journey !== undefined);
|
|
411
|
+
}
|
|
412
|
+
export function normalizeVerifyJourney(raw, baseDir = process.cwd()) {
|
|
413
|
+
const limits = VERIFY_JOURNEY_LIMITS;
|
|
414
|
+
if (!isRecord(raw))
|
|
415
|
+
throw new Error('flow must be a JSON object.');
|
|
416
|
+
const extra = Object.keys(raw).filter((key) => key !== 'actors' && key !== 'journey');
|
|
417
|
+
if (extra.length) {
|
|
418
|
+
throw new Error(`A multi-user flow has only "actors" and "journey" (found ${extra.join(', ')}). Put actions, path, viewport and expect_requests on each journey segment, and session seeds on each actor.`);
|
|
419
|
+
}
|
|
420
|
+
if (!isRecord(raw.actors))
|
|
421
|
+
throw new Error('actors must be an object such as { "alice": {}, "bob": { "auth": { "user_id": "..." } } }.');
|
|
422
|
+
const names = Object.keys(raw.actors);
|
|
423
|
+
if (names.length < 1 || names.length > limits.actors) {
|
|
424
|
+
throw new Error(`actors has ${names.length} entries; a journey has 1 to ${limits.actors} actors, each its own browser.`);
|
|
425
|
+
}
|
|
426
|
+
const actors = {};
|
|
427
|
+
for (const name of names) {
|
|
428
|
+
if (!ACTOR_NAME.test(name))
|
|
429
|
+
throw new Error(`actor name "${name}" must be 1-16 lowercase letters, digits, "-" or "_", starting with a letter.`);
|
|
430
|
+
const value = raw.actors[name];
|
|
431
|
+
if (!isRecord(value))
|
|
432
|
+
throw new Error(`actors.${name} must be an object ({} for a signed-out browser).`);
|
|
433
|
+
const unknown = Object.keys(value).filter((key) => !['auth', 'local_storage', 'cookies', 'headers'].includes(key));
|
|
434
|
+
if (unknown.length)
|
|
435
|
+
throw new Error(`actors.${name} has unsupported field${unknown.length === 1 ? '' : 's'}: ${unknown.join(', ')}.`);
|
|
436
|
+
const actor = {};
|
|
437
|
+
const auth = normalizeAuth(value.auth);
|
|
438
|
+
const localStorage = normalizeStringRecord(value.local_storage, `actors.${name}.local_storage`);
|
|
439
|
+
const cookies = normalizeCookies(value.cookies);
|
|
440
|
+
const headers = normalizeStringRecord(value.headers, `actors.${name}.headers`);
|
|
441
|
+
if (auth)
|
|
442
|
+
actor.auth = auth;
|
|
443
|
+
if (localStorage)
|
|
444
|
+
actor.local_storage = localStorage;
|
|
445
|
+
if (cookies)
|
|
446
|
+
actor.cookies = cookies;
|
|
447
|
+
if (headers)
|
|
448
|
+
actor.headers = headers;
|
|
449
|
+
assertSessionSeedSize(actor);
|
|
450
|
+
actors[name] = actor;
|
|
451
|
+
}
|
|
452
|
+
if (!Array.isArray(raw.journey) || raw.journey.length < 1)
|
|
453
|
+
throw new Error('journey must be a non-empty array of segments like { "as": "alice", "path": "/", "actions": [...] }.');
|
|
454
|
+
if (raw.journey.length > limits.segments) {
|
|
455
|
+
throw new Error(`journey has ${raw.journey.length} segments; the limit is ${limits.segments}. Each segment is one browser run, so combine consecutive steps by the same actor.`);
|
|
456
|
+
}
|
|
457
|
+
const seen = new Set();
|
|
458
|
+
let total = 0;
|
|
459
|
+
const journey = raw.journey.map((value, index) => {
|
|
460
|
+
const at = `journey[${index}]`;
|
|
461
|
+
if (!isRecord(value))
|
|
462
|
+
throw new Error(`${at} must be an object.`);
|
|
463
|
+
const unknown = Object.keys(value).filter((key) => !['as', 'path', 'actions', 'expect_requests', 'visible_only', 'viewport'].includes(key));
|
|
464
|
+
if (unknown.length)
|
|
465
|
+
throw new Error(`${at} has unsupported field${unknown.length === 1 ? '' : 's'}: ${unknown.join(', ')}.`);
|
|
466
|
+
if (typeof value.as !== 'string' || !(value.as in actors)) {
|
|
467
|
+
throw new Error(`${at}.as must name one of the actors (${names.join(', ')}).`);
|
|
468
|
+
}
|
|
469
|
+
if (value.path !== undefined && !isJourneyPath(value.path)) {
|
|
470
|
+
throw new Error(`${at}.path must be a path on this app starting with "/" (for example "/club").`);
|
|
471
|
+
}
|
|
472
|
+
const actions = normalizeBrowserActions(value.actions ?? [], baseDir);
|
|
473
|
+
if (!actions.ok)
|
|
474
|
+
throw new Error(`${at}: ${actions.error}`);
|
|
475
|
+
if (value.visible_only !== undefined && typeof value.visible_only !== 'boolean')
|
|
476
|
+
throw new Error(`${at}.visible_only must be boolean.`);
|
|
477
|
+
const continued = seen.has(value.as);
|
|
478
|
+
seen.add(value.as);
|
|
479
|
+
// A continued actor's page moves with a leading goto, which is one action.
|
|
480
|
+
const count = actions.actions.length + (continued && value.path ? 1 : 0);
|
|
481
|
+
if (count > limits.actions_per_segment) {
|
|
482
|
+
throw new Error(`${at} has ${count} actions${continued && value.path ? ' (including the goto for its path)' : ''}; the limit per segment is ${limits.actions_per_segment}. Split it into two segments by the same actor.`);
|
|
483
|
+
}
|
|
484
|
+
total += count;
|
|
485
|
+
const viewport = value.viewport === undefined ? 'desktop' : normalizeViewports([value.viewport])[0];
|
|
486
|
+
return {
|
|
487
|
+
as: value.as,
|
|
488
|
+
...(value.path !== undefined ? { path: value.path } : {}),
|
|
489
|
+
actions: actions.actions,
|
|
490
|
+
expect_requests: normalizeExpectedRequests(value.expect_requests),
|
|
491
|
+
visible_only: value.visible_only === true,
|
|
492
|
+
viewport,
|
|
493
|
+
};
|
|
494
|
+
});
|
|
495
|
+
if (total > limits.total_actions)
|
|
496
|
+
throw new Error(`journey has ${total} actions in total; the limit is ${limits.total_actions}.`);
|
|
497
|
+
const unused = names.filter((name) => !seen.has(name));
|
|
498
|
+
if (unused.length)
|
|
499
|
+
throw new Error(`actor${unused.length === 1 ? '' : 's'} ${unused.join(', ')} ${unused.length === 1 ? 'is' : 'are'} declared but never used in the journey.`);
|
|
500
|
+
return { actors, journey };
|
|
501
|
+
}
|
|
502
|
+
export function loadVerifyInput(path, cwd = process.cwd()) {
|
|
503
|
+
if (!path)
|
|
504
|
+
return { kind: 'single', flow: normalizeVerifyFlow(undefined) };
|
|
505
|
+
const absolute = resolve(cwd, path);
|
|
506
|
+
if (!existsSync(absolute))
|
|
507
|
+
throw new Error(`Flow file not found: ${absolute}`);
|
|
508
|
+
let parsed;
|
|
509
|
+
try {
|
|
510
|
+
parsed = JSON.parse(readFileSync(absolute, 'utf8'));
|
|
511
|
+
}
|
|
512
|
+
catch (cause) {
|
|
513
|
+
throw new Error(`Flow file is not valid JSON: ${absolute} — ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
514
|
+
}
|
|
515
|
+
return isVerifyJourneyInput(parsed)
|
|
516
|
+
? { kind: 'journey', journey: normalizeVerifyJourney(parsed, dirname(absolute)) }
|
|
517
|
+
: { kind: 'single', flow: normalizeVerifyFlow(parsed, dirname(absolute)) };
|
|
518
|
+
}
|
|
519
|
+
const UNCONFIRMED_BACKSTOP = 'the platform closes an idle browser within 3 minutes';
|
|
520
|
+
/** A call the client stopped waiting for may still be admitted on the
|
|
521
|
+
* platform; wait this long (inside the cleanup budget) before closing again. */
|
|
522
|
+
const RECLOSE_AFTER_ABANDONED_MS = 5_000;
|
|
523
|
+
function settleWithin(promise, ms) {
|
|
524
|
+
if (ms <= 0)
|
|
525
|
+
return Promise.resolve(false);
|
|
526
|
+
return new Promise((resolveSettled) => {
|
|
527
|
+
const timer = setTimeout(() => resolveSettled(false), ms);
|
|
528
|
+
promise.then(() => { clearTimeout(timer); resolveSettled(true); }, () => { clearTimeout(timer); resolveSettled(true); });
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
export function createVerifyJourneyRun(target, journey, client, opts = {}) {
|
|
532
|
+
const now = opts.now ?? Date.now;
|
|
533
|
+
const cleanupBudgetMs = opts.cleanupBudgetMs ?? VERIFY_JOURNEY_LIMITS.cleanup_budget_ms;
|
|
534
|
+
const recloseAfterAbandonedMs = opts.recloseAfterAbandonedMs ?? RECLOSE_AFTER_ABANDONED_MS;
|
|
535
|
+
const runId = opts.runId ?? randomBytes(4).toString('hex');
|
|
536
|
+
const sessions = Object.fromEntries(Object.keys(journey.actors).map((name) => [name, `vf-${runId}-${name}`]));
|
|
537
|
+
// Actors a call has been sent for (their browser may exist).
|
|
538
|
+
const opened = new Set();
|
|
539
|
+
// Actors whose last call ended without a platform answer (client timeout or
|
|
540
|
+
// lost connection): the platform may still be running it.
|
|
541
|
+
const abandoned = new Set();
|
|
542
|
+
// The one segment call in flight, if any.
|
|
543
|
+
let inFlight = null;
|
|
544
|
+
let stopReason = null;
|
|
545
|
+
let closing = null;
|
|
546
|
+
const stop = (code, message) => {
|
|
547
|
+
stopReason ??= { code, message };
|
|
548
|
+
};
|
|
549
|
+
const closeOne = async (actor, budgetEnd) => {
|
|
550
|
+
const remaining = budgetEnd - now();
|
|
551
|
+
if (remaining <= 0)
|
|
552
|
+
return { error: 'cleanup budget spent' };
|
|
553
|
+
try {
|
|
554
|
+
const result = await client.call('POST', '/browser/test', { session_id: sessions[actor] }, undefined, { timeoutMs: remaining });
|
|
555
|
+
return { closed: result?.closed === true };
|
|
556
|
+
}
|
|
557
|
+
catch (cause) {
|
|
558
|
+
return { error: cause instanceof CliApiError ? cause.code : cause instanceof Error ? cause.message : String(cause) };
|
|
559
|
+
}
|
|
560
|
+
};
|
|
561
|
+
const closeAll = () => {
|
|
562
|
+
// Set before anything is awaited, so a segment that settles while the
|
|
563
|
+
// browsers are closing cannot start another one.
|
|
564
|
+
stop('VERIFY_STOPPED', 'The run stopped to close its browsers.');
|
|
565
|
+
closing ??= (async () => {
|
|
566
|
+
const startedAt = now();
|
|
567
|
+
const budgetEnd = startedAt + cleanupBudgetMs;
|
|
568
|
+
const pending = inFlight;
|
|
569
|
+
const actors = [...opened];
|
|
570
|
+
// One concurrent pass: every opened browser at once, one shared budget.
|
|
571
|
+
const first = await Promise.all(actors.map(async (actor) => [actor, await closeOne(actor, budgetEnd)]));
|
|
572
|
+
const outcome = new Map();
|
|
573
|
+
for (const [actor, result] of first) {
|
|
574
|
+
outcome.set(actor, 'error' in result
|
|
575
|
+
? { actor, status: 'unconfirmed', reason: `close failed (${result.error}); ${UNCONFIRMED_BACKSTOP}` }
|
|
576
|
+
: { actor, status: result.closed ? 'closed' : 'already_closed' });
|
|
577
|
+
}
|
|
578
|
+
// A call still in flight may create or reuse its browser after the close
|
|
579
|
+
// above. Wait for it inside the budget, then close that actor again.
|
|
580
|
+
if (pending) {
|
|
581
|
+
const settled = await settleWithin(pending.settled, budgetEnd - now());
|
|
582
|
+
if (!settled) {
|
|
583
|
+
outcome.set(pending.actor, { actor: pending.actor, status: 'unconfirmed', reason: `its call was still running when cleanup ran out of time; ${UNCONFIRMED_BACKSTOP}` });
|
|
584
|
+
}
|
|
585
|
+
else if (!abandoned.has(pending.actor)) {
|
|
586
|
+
const again = await closeOne(pending.actor, budgetEnd);
|
|
587
|
+
const earlier = outcome.get(pending.actor);
|
|
588
|
+
outcome.set(pending.actor, 'error' in again
|
|
589
|
+
? { actor: pending.actor, status: 'unconfirmed', reason: `close after its call finished failed (${again.error}); ${UNCONFIRMED_BACKSTOP}` }
|
|
590
|
+
: { actor: pending.actor, status: again.closed || earlier?.status === 'closed' ? 'closed' : 'already_closed' });
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
// A call the client gave up on has no answer to wait for. Close once more
|
|
594
|
+
// a little later (inside the budget) and say plainly it is unconfirmed.
|
|
595
|
+
for (const actor of actors.filter((name) => abandoned.has(name))) {
|
|
596
|
+
const waitMs = Math.min(recloseAfterAbandonedMs, budgetEnd - now());
|
|
597
|
+
if (waitMs > 0)
|
|
598
|
+
await new Promise((resolveWait) => setTimeout(resolveWait, waitMs));
|
|
599
|
+
const again = await closeOne(actor, budgetEnd);
|
|
600
|
+
outcome.set(actor, {
|
|
601
|
+
actor,
|
|
602
|
+
status: 'unconfirmed',
|
|
603
|
+
reason: `its call got no answer, so the platform may still start it${'error' in again ? ` (the second close failed: ${again.error})` : ''}; ${UNCONFIRMED_BACKSTOP}`,
|
|
604
|
+
});
|
|
605
|
+
}
|
|
606
|
+
return { cleanup: actors.map((actor) => outcome.get(actor)), cleanup_ms: now() - startedAt };
|
|
607
|
+
})();
|
|
608
|
+
return closing;
|
|
609
|
+
};
|
|
610
|
+
const run = async () => {
|
|
611
|
+
if (!target.project_id) {
|
|
612
|
+
throw new Error('Multi-user verification needs --project <project>: every actor signs in to that project, and its browsers stay on its addresses.');
|
|
613
|
+
}
|
|
614
|
+
if (target.url && isLoopbackUrl(target.url)) {
|
|
615
|
+
throw new Error('Multi-user journeys run on the deployed app. Deploy first, then run somewhere verify --project <project> --flow <file>.');
|
|
616
|
+
}
|
|
617
|
+
const startedAt = now();
|
|
618
|
+
const deadlineAt = startedAt + VERIFY_JOURNEY_LIMITS.deadline_ms;
|
|
619
|
+
const segments = [];
|
|
620
|
+
const runs = [];
|
|
621
|
+
const started = new Set();
|
|
622
|
+
// 'report': the segment ran and its own report failed (the step verdict
|
|
623
|
+
// says why). 'error': the segment could not run or could not continue.
|
|
624
|
+
let stopped = null;
|
|
625
|
+
const deadlineMessage = `The journey reached its ${VERIFY_JOURNEY_LIMITS.deadline_ms / 60000}-minute limit.`;
|
|
626
|
+
try {
|
|
627
|
+
for (const [index, segment] of journey.journey.entries()) {
|
|
628
|
+
const viewport = resolveViewport(segment.viewport);
|
|
629
|
+
const label = `${segment.as} #${index + 1} ${viewport.label}`;
|
|
630
|
+
const base = {
|
|
631
|
+
segment: index + 1,
|
|
632
|
+
as: segment.as,
|
|
633
|
+
viewport: viewport.label,
|
|
634
|
+
...(segment.path ? { path: segment.path } : {}),
|
|
635
|
+
ran: false,
|
|
636
|
+
passed: false,
|
|
637
|
+
};
|
|
638
|
+
if (!stopped && !stopReason && now() >= deadlineAt)
|
|
639
|
+
stop('VERIFY_JOURNEY_DEADLINE', deadlineMessage);
|
|
640
|
+
if (stopped) {
|
|
641
|
+
segments.push(base);
|
|
642
|
+
continue;
|
|
643
|
+
}
|
|
644
|
+
if (stopReason) {
|
|
645
|
+
stopped = { kind: 'error', segment: index + 1, as: segment.as, viewport: viewport.label, ...stopReason };
|
|
646
|
+
segments.push({ ...base, error: { ...stopReason } });
|
|
647
|
+
continue;
|
|
648
|
+
}
|
|
649
|
+
const actor = journey.actors[segment.as];
|
|
650
|
+
const first = !started.has(segment.as);
|
|
651
|
+
const actions = !first && segment.path ? [{ goto: segment.path }, ...segment.actions] : segment.actions;
|
|
652
|
+
const hasSeeds = actor.local_storage !== undefined || actor.cookies !== undefined || actor.headers !== undefined;
|
|
653
|
+
const url = first ? (segment.path ?? target.url ?? (hasSeeds ? '/' : undefined)) : undefined;
|
|
654
|
+
const body = {
|
|
655
|
+
project_id: target.project_id,
|
|
656
|
+
session_id: sessions[segment.as],
|
|
657
|
+
...(url ? { url } : {}),
|
|
658
|
+
actions,
|
|
659
|
+
expect_requests: segment.expect_requests,
|
|
660
|
+
visible_only: segment.visible_only,
|
|
661
|
+
viewport: viewport.wire,
|
|
662
|
+
capture_after: true,
|
|
663
|
+
inline: false,
|
|
664
|
+
...(first && actor.auth ? { auth: actor.auth } : {}),
|
|
665
|
+
...(first && actor.local_storage ? { local_storage: actor.local_storage } : {}),
|
|
666
|
+
...(first && actor.cookies ? { cookies: actor.cookies } : {}),
|
|
667
|
+
...(first && actor.headers ? { headers: actor.headers } : {}),
|
|
668
|
+
};
|
|
669
|
+
// Each wait is capped by what is left of the journey budget.
|
|
670
|
+
const remaining = deadlineAt - now();
|
|
671
|
+
const waitMs = Math.min(VERIFY_JOURNEY_LIMITS.segment_timeout_ms, remaining);
|
|
672
|
+
opened.add(segment.as);
|
|
673
|
+
started.add(segment.as);
|
|
674
|
+
abandoned.delete(segment.as);
|
|
675
|
+
const call = client.call('POST', '/browser/test', body, undefined, { timeoutMs: waitMs });
|
|
676
|
+
inFlight = { actor: segment.as, settled: call };
|
|
677
|
+
let report;
|
|
678
|
+
try {
|
|
679
|
+
report = await call;
|
|
680
|
+
}
|
|
681
|
+
catch (cause) {
|
|
682
|
+
const code = cause instanceof CliApiError ? cause.code : 'VERIFY_SEGMENT_FAILED';
|
|
683
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
684
|
+
if (code === 'TIMEOUT' || code === 'NETWORK_ERROR')
|
|
685
|
+
abandoned.add(segment.as);
|
|
686
|
+
const budgetCut = code === 'TIMEOUT' && waitMs < VERIFY_JOURNEY_LIMITS.segment_timeout_ms;
|
|
687
|
+
const reported = budgetCut ? { code: 'VERIFY_JOURNEY_DEADLINE', message: `${deadlineMessage} Segment ${index + 1} was still running.` } : { code, message };
|
|
688
|
+
if (budgetCut)
|
|
689
|
+
stop(reported.code, reported.message);
|
|
690
|
+
stopped = { kind: 'error', segment: index + 1, as: segment.as, viewport: viewport.label, ...reported };
|
|
691
|
+
// A refusal (limit, authority, validation) happens before a browser runs.
|
|
692
|
+
segments.push({ ...base, error: reported });
|
|
693
|
+
continue;
|
|
694
|
+
}
|
|
695
|
+
finally {
|
|
696
|
+
inFlight = null;
|
|
697
|
+
}
|
|
698
|
+
runs.push({ viewport: { ...viewport, label }, report, actions });
|
|
699
|
+
// A continued actor whose browser was reaped would carry on signed out
|
|
700
|
+
// in a fresh browser; that is a different person, so the journey stops.
|
|
701
|
+
if (!first && report.session_note) {
|
|
702
|
+
const message = `${segment.as}'s browser ended between segments (${report.session_note}), so ${segment.as} is no longer the same signed-in user. Keep each actor's gap between segments under 3 minutes and run the journey again.`;
|
|
703
|
+
stopped = { kind: 'error', segment: index + 1, as: segment.as, viewport: viewport.label, code: 'VERIFY_ACTOR_SESSION_ENDED', message };
|
|
704
|
+
segments.push({ ...base, ran: true, final_url: report.final_url, error: { code: stopped.code, message } });
|
|
705
|
+
continue;
|
|
706
|
+
}
|
|
707
|
+
const passed = reportPassed(report);
|
|
708
|
+
segments.push({ ...base, ran: true, passed, ...(report.final_url ? { final_url: report.final_url } : {}) });
|
|
709
|
+
if (!passed)
|
|
710
|
+
stopped = { kind: 'report', segment: index + 1, as: segment.as, viewport: viewport.label, code: '', message: '' };
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
finally {
|
|
714
|
+
await closeAll();
|
|
715
|
+
}
|
|
716
|
+
const shaped = shapeVerificationReport(target.url ?? target.project_id, { actions: [] }, runs);
|
|
717
|
+
const names = Object.keys(journey.actors);
|
|
718
|
+
const passed = !stopped && runs.length === journey.journey.length && shaped.passed;
|
|
719
|
+
let verdict;
|
|
720
|
+
if (stopped?.kind === 'error') {
|
|
721
|
+
verdict = `FAIL — segment ${stopped.segment} (${stopped.as}, ${stopped.viewport}): ${stopped.message} [${stopped.code}]`;
|
|
722
|
+
}
|
|
723
|
+
else if (!passed) {
|
|
724
|
+
verdict = shaped.verdict.replace(/^FAIL — /, `FAIL — segment ${stopped?.segment ?? '?'} — `);
|
|
725
|
+
}
|
|
726
|
+
else {
|
|
727
|
+
verdict = `PASS — ${journey.journey.length} segment${journey.journey.length === 1 ? '' : 's'} by ${names.length} user${names.length === 1 ? '' : 's'} (${names.join(', ')}) passed; page, console, and network healthy.`;
|
|
728
|
+
}
|
|
729
|
+
const { cleanup, cleanup_ms } = await closeAll();
|
|
730
|
+
return {
|
|
731
|
+
...shaped,
|
|
732
|
+
passed,
|
|
733
|
+
verdict,
|
|
734
|
+
mode: 'journey',
|
|
735
|
+
actors: names,
|
|
736
|
+
segments,
|
|
737
|
+
browser_runs: segments.filter((segment) => segment.ran).length,
|
|
738
|
+
limits: VERIFY_JOURNEY_LIMITS,
|
|
739
|
+
cleanup,
|
|
740
|
+
cleanup_ms,
|
|
741
|
+
cleanup_confirmed: cleanup.every((item) => item.status !== 'unconfirmed'),
|
|
742
|
+
};
|
|
743
|
+
};
|
|
744
|
+
return { sessions, run, stop, closeAll };
|
|
745
|
+
}
|
|
746
|
+
/** The flow file contract, for agents: `somewhere verify --schema`. */
|
|
747
|
+
export function verifyFlowSchema() {
|
|
748
|
+
const L = VERIFY_JOURNEY_LIMITS;
|
|
749
|
+
const path = { type: 'string', pattern: '^/(?!/)[^\\s\\\\]*$', description: 'A path on the app, e.g. "/club".' };
|
|
750
|
+
const action = {
|
|
751
|
+
description: 'Exactly one action key per item.',
|
|
752
|
+
oneOf: [
|
|
753
|
+
{ type: 'object', required: ['click'], properties: { click: { type: 'string', description: 'CSS selector' } }, additionalProperties: false },
|
|
754
|
+
{ type: 'object', required: ['fill', 'value'], properties: { fill: { type: 'string' }, value: { type: 'string' } }, additionalProperties: false },
|
|
755
|
+
{ type: 'object', required: ['select', 'value'], properties: { select: { type: 'string' }, value: { type: 'string' } }, additionalProperties: false },
|
|
756
|
+
{ type: 'object', required: ['upload', 'file'], properties: { upload: { type: 'string' }, file: { type: 'string', description: 'Local path, data URL, or base64' }, name: { type: 'string' } }, additionalProperties: false },
|
|
757
|
+
{ type: 'object', required: ['wait'], properties: { wait: { oneOf: [{ type: 'string' }, { type: 'number', minimum: 0 }, { type: 'object', required: ['selector'], properties: { selector: { type: 'string' } } }] } }, additionalProperties: false },
|
|
758
|
+
{ type: 'object', required: ['expect'], properties: { expect: { type: 'object', required: ['selector'], properties: { selector: { type: 'string' }, text: { type: 'string' }, value: { type: 'string' }, visible: { type: 'boolean' }, count: { type: 'integer', minimum: 0 } } } }, additionalProperties: false },
|
|
759
|
+
{ type: 'object', required: ['screenshot'], properties: { screenshot: { type: 'string', description: 'Label' } }, additionalProperties: false },
|
|
760
|
+
{ type: 'object', required: ['eval'], properties: { eval: { type: 'string', description: 'Page script on your own app; its value is reported and a throw fails the step. Use it for in-page API checks.' } }, additionalProperties: false },
|
|
761
|
+
{ type: 'object', required: ['goto'], properties: { goto: path }, additionalProperties: false },
|
|
762
|
+
],
|
|
763
|
+
};
|
|
764
|
+
const expectRequests = { type: 'array', maxItems: 30, items: { type: 'object', required: ['path', 'status'], properties: { path: { type: 'string' }, status: { type: 'integer', minimum: 100, maximum: 599 } }, additionalProperties: false }, description: 'Responses the flow intends, such as a 403 for another user. Seen = not a failure; missing = failure.' };
|
|
765
|
+
const viewport = { oneOf: [{ enum: ['desktop', 'mobile'] }, { type: 'object', required: ['label', 'width', 'height'], properties: { label: { type: 'string' }, width: { type: 'integer', minimum: 100, maximum: 3840 }, height: { type: 'integer', minimum: 100, maximum: 2160 } } }] };
|
|
766
|
+
const seeds = {
|
|
767
|
+
auth: { type: 'object', required: ['user_id'], properties: { user_id: { type: 'string', description: 'An existing app user of this project to sign in as.' } }, additionalProperties: false },
|
|
768
|
+
local_storage: { type: 'object', additionalProperties: { type: 'string' } },
|
|
769
|
+
cookies: { type: 'array', items: { type: 'object', required: ['name', 'value'], properties: { name: { type: 'string' }, value: { type: 'string' } } } },
|
|
770
|
+
headers: { type: 'object', additionalProperties: { type: 'string' } },
|
|
771
|
+
};
|
|
772
|
+
return {
|
|
773
|
+
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
|
774
|
+
title: 'somewhere verify --flow',
|
|
775
|
+
oneOf: [
|
|
776
|
+
{
|
|
777
|
+
title: 'One user',
|
|
778
|
+
type: 'object',
|
|
779
|
+
properties: { actions: { type: 'array', maxItems: 30, items: action }, ...seeds, expect_requests: expectRequests, visible_only: { type: 'boolean' }, viewports: { type: 'array', minItems: 1, maxItems: 4, items: viewport } },
|
|
780
|
+
additionalProperties: false,
|
|
781
|
+
},
|
|
782
|
+
{
|
|
783
|
+
title: 'Several users (needs --project; hosted apps; claimed accounts)',
|
|
784
|
+
type: 'object',
|
|
785
|
+
required: ['actors', 'journey'],
|
|
786
|
+
properties: {
|
|
787
|
+
actors: {
|
|
788
|
+
type: 'object', minProperties: 1, maxProperties: L.actors,
|
|
789
|
+
propertyNames: { pattern: ACTOR_NAME.source },
|
|
790
|
+
additionalProperties: { type: 'object', properties: seeds, additionalProperties: false, description: 'Each actor is its own browser. {} starts signed out.' },
|
|
791
|
+
},
|
|
792
|
+
journey: {
|
|
793
|
+
type: 'array', minItems: 1, maxItems: L.segments,
|
|
794
|
+
items: {
|
|
795
|
+
type: 'object', required: ['as'],
|
|
796
|
+
properties: {
|
|
797
|
+
as: { type: 'string', description: 'Actor name' },
|
|
798
|
+
path: { ...path, description: 'Where this segment starts: the actor\'s first page, or a goto for a continued actor.' },
|
|
799
|
+
actions: { type: 'array', maxItems: L.actions_per_segment, items: action },
|
|
800
|
+
expect_requests: expectRequests,
|
|
801
|
+
viewport: { ...viewport, default: 'desktop' },
|
|
802
|
+
visible_only: { type: 'boolean' },
|
|
803
|
+
},
|
|
804
|
+
additionalProperties: false,
|
|
805
|
+
},
|
|
806
|
+
},
|
|
807
|
+
},
|
|
808
|
+
additionalProperties: false,
|
|
809
|
+
},
|
|
810
|
+
],
|
|
811
|
+
'x-limits': L,
|
|
812
|
+
};
|
|
813
|
+
}
|
|
374
814
|
export function formatVerifyReport(report) {
|
|
375
815
|
const lines = [report.passed ? green(report.verdict) : red(report.verdict)];
|
|
376
816
|
for (const step of report.steps) {
|
|
@@ -385,6 +825,25 @@ export function formatVerifyReport(report) {
|
|
|
385
825
|
if (shot.fs_path && (shot.url || shot.scratch_url))
|
|
386
826
|
lines.push(`screenshot_file: [${shot.viewport}] ${shot.fs_path}`);
|
|
387
827
|
}
|
|
828
|
+
for (const item of report.layout ?? [])
|
|
829
|
+
lines.push(`layout: [${item.viewport}] ${dim(item.detail)}`);
|
|
830
|
+
return lines;
|
|
831
|
+
}
|
|
832
|
+
export function formatVerifyJourneyReport(report) {
|
|
833
|
+
const lines = formatVerifyReport(report);
|
|
834
|
+
const segmentLines = report.segments.map((segment) => {
|
|
835
|
+
const mark = !segment.ran && !segment.error ? dim('not run') : segment.passed ? green('✓') : red('✗');
|
|
836
|
+
return `segment ${segment.segment} ${mark} ${segment.as} [${segment.viewport}]${segment.path ? ` ${segment.path}` : ''}${segment.error ? ` ${dim(`— ${segment.error.code}`)}` : ''}`;
|
|
837
|
+
});
|
|
838
|
+
return [lines[0], ...segmentLines, ...lines.slice(1), `browser runs used: ${report.browser_runs}`, ...formatJourneyCleanup(report.cleanup, report.cleanup_ms)];
|
|
839
|
+
}
|
|
840
|
+
export function formatJourneyCleanup(cleanup, cleanupMs) {
|
|
841
|
+
if (!cleanup.length)
|
|
842
|
+
return ['browsers: none were opened'];
|
|
843
|
+
const lines = [`browsers (cleanup ${Math.round(cleanupMs / 100) / 10}s of ${VERIFY_JOURNEY_LIMITS.cleanup_budget_ms / 1000}s budget): ${cleanup.map((item) => `${item.actor} ${item.status.replace('_', ' ')}`).join(', ')}`];
|
|
844
|
+
for (const item of cleanup)
|
|
845
|
+
if (item.status === 'unconfirmed' && item.reason)
|
|
846
|
+
lines.push(` ${item.actor}: ${item.reason}`);
|
|
388
847
|
return lines;
|
|
389
848
|
}
|
|
390
849
|
export function registerVerify(program) {
|
|
@@ -405,6 +864,7 @@ export function registerVerify(program) {
|
|
|
405
864
|
.option('--session <session-id>', 'Existing app session value to seed as localStorage sw_auth in every viewport.')
|
|
406
865
|
.option('--cookie <name=value>', 'Existing app cookie to seed in every viewport. Repeatable.', collectCookie)
|
|
407
866
|
.option('--json', 'Print the structured verification report as JSON.')
|
|
867
|
+
.option('--schema', 'Print the JSON Schema for --flow files (one user, or several users) and exit.')
|
|
408
868
|
.addHelpText('after', `
|
|
409
869
|
Minimal --flow JSON:
|
|
410
870
|
{
|
|
@@ -418,8 +878,33 @@ Minimal --flow JSON:
|
|
|
418
878
|
|
|
419
879
|
Save that object as flow.json, then pass --flow flow.json.
|
|
420
880
|
For the complete flow and action schema, run: somewhere docs browser
|
|
881
|
+
|
|
882
|
+
Several users in one run (--project required; each actor is its own browser):
|
|
883
|
+
{
|
|
884
|
+
"actors": { "alice": {}, "bob": {} },
|
|
885
|
+
"journey": [
|
|
886
|
+
{ "as": "alice", "path": "/signup", "actions": [{ "click": "#create" }] },
|
|
887
|
+
{ "as": "bob", "path": "/club", "actions": [{ "click": "#join" }],
|
|
888
|
+
"expect_requests": [{ "path": "/api/admin", "status": 403 }] },
|
|
889
|
+
{ "as": "alice", "path": "/club", "viewport": "mobile",
|
|
890
|
+
"actions": [{ "eval": "fetch('/api/state').then(r => r.status)" }] }
|
|
891
|
+
]
|
|
892
|
+
}
|
|
893
|
+
Segments run in order; a later segment for the same actor continues in that
|
|
894
|
+
actor's browser (path becomes a goto). Limits: ${VERIFY_JOURNEY_LIMITS.actors} actors, ${VERIFY_JOURNEY_LIMITS.segments} segments,
|
|
895
|
+
${VERIFY_JOURNEY_LIMITS.actions_per_segment} actions per segment, ${VERIFY_JOURNEY_LIMITS.total_actions} in total, ${VERIFY_JOURNEY_LIMITS.deadline_ms / 60000} minutes. Each segment is one browser
|
|
896
|
+
run against your plan's browser limits. When the run ends or is interrupted,
|
|
897
|
+
every actor browser is closed within ${VERIFY_JOURNEY_LIMITS.cleanup_budget_ms / 1000} seconds; any close that cannot be
|
|
898
|
+
confirmed is reported, and the platform closes idle browsers within 3 minutes.
|
|
899
|
+
Several users need a claimed account and a CLI login approved for all your
|
|
900
|
+
projects: a login approved for "Only these projects" cannot open named browsers
|
|
901
|
+
(SESSION_SCOPE_FORBIDDEN). Print the full flow schema with: somewhere verify --schema
|
|
421
902
|
`)
|
|
422
903
|
.action(async (target, opts) => {
|
|
904
|
+
if (opts.schema) {
|
|
905
|
+
console.log(JSON.stringify(verifyFlowSchema(), null, 2));
|
|
906
|
+
process.exit(0);
|
|
907
|
+
}
|
|
423
908
|
try {
|
|
424
909
|
const url = opts.url ?? (target && /^https?:\/\//i.test(target) ? target : undefined);
|
|
425
910
|
const local = !!url && isLoopbackUrl(url);
|
|
@@ -450,7 +935,44 @@ For the complete flow and action schema, run: somewhere docs browser
|
|
|
450
935
|
}
|
|
451
936
|
if (!local && !client)
|
|
452
937
|
client = new ApiClient(getToken());
|
|
453
|
-
const
|
|
938
|
+
const input = loadVerifyInput(opts.flow);
|
|
939
|
+
if (input.kind === 'journey') {
|
|
940
|
+
if (opts.session || cliCookies.length) {
|
|
941
|
+
throw new Error('--session and --cookie seed one browser; in a multi-user flow put each actor\'s seeds on that actor in the flow file.');
|
|
942
|
+
}
|
|
943
|
+
if (!client)
|
|
944
|
+
client = new ApiClient(getToken());
|
|
945
|
+
const journeyRun = createVerifyJourneyRun({ ...(project ? { project_id: project } : {}), ...(url ? { url } : {}) }, input.journey, client);
|
|
946
|
+
// Ctrl-C or a terminating signal: stop starting segments at once, then
|
|
947
|
+
// close every opened browser within the cleanup budget and say what
|
|
948
|
+
// could not be confirmed.
|
|
949
|
+
const onSignal = (signal) => {
|
|
950
|
+
journeyRun.stop('VERIFY_INTERRUPTED', `Stopped by ${signal}.`);
|
|
951
|
+
void journeyRun.closeAll()
|
|
952
|
+
.then(({ cleanup, cleanup_ms }) => {
|
|
953
|
+
for (const line of formatJourneyCleanup(cleanup, cleanup_ms))
|
|
954
|
+
process.stderr.write(`${line}\n`);
|
|
955
|
+
})
|
|
956
|
+
.finally(() => process.exit(signal === 'SIGINT' ? 130 : 143));
|
|
957
|
+
};
|
|
958
|
+
process.once('SIGINT', onSignal);
|
|
959
|
+
process.once('SIGTERM', onSignal);
|
|
960
|
+
let report;
|
|
961
|
+
try {
|
|
962
|
+
report = await journeyRun.run();
|
|
963
|
+
}
|
|
964
|
+
finally {
|
|
965
|
+
process.removeListener('SIGINT', onSignal);
|
|
966
|
+
process.removeListener('SIGTERM', onSignal);
|
|
967
|
+
}
|
|
968
|
+
if (opts.json)
|
|
969
|
+
console.log(JSON.stringify(report, null, 2));
|
|
970
|
+
else
|
|
971
|
+
for (const line of formatVerifyJourneyReport(report))
|
|
972
|
+
console.log(line);
|
|
973
|
+
process.exit(report.passed ? 0 : 1);
|
|
974
|
+
}
|
|
975
|
+
const loadedFlow = input.flow;
|
|
454
976
|
const flow = {
|
|
455
977
|
...loadedFlow,
|
|
456
978
|
...(opts.session ? { local_storage: { ...loadedFlow.local_storage, sw_auth: opts.session } } : {}),
|