@sifurai/cli 0.1.2 → 0.1.4

package/dist/evidence.js

@@ -0,0 +1,156 @@
+import { redact } from './secrets.js';
+export const MAX_TEXT = 20_000;
+export const MAX_ELEMENTS = 40;
+export const MAX_NAME = 120;
+// The API's limits: one value over them and the whole upload is refused.
+export const MAX_SELECTOR = 512;
+export const MAX_URL = 2048;
+const CAPTURE_TIMEOUT = 10_000;
+const SENSITIVE_KEY = /token|passw|pwd|secret|session|key|auth|signature|jwt|otp|^sid$|^code$|^sig$/i;
+/** The page's address with secrets taken out, and the values of query (or fragment) keys that usually carry one. */
+export function redactUrl(url, secrets) {
+    let redacted = url;
+    for (const value of Object.values(secrets)) {
+        if (!value)
+            continue;
+        redacted = redacted.replaceAll(value, '[secret]');
+        const encoded = encodeURIComponent(value);
+        if (encoded !== value)
+            redacted = redacted.replaceAll(encoded, '[secret]');
+    }
+    return redacted.replace(/([?&#;])([^=&#;?]+)=([^&#;]*)/g, (match, separator, key) => {
+        let name = key;
+        try {
+            name = decodeURIComponent(key);
+        }
+        catch { }
+        return SENSITIVE_KEY.test(name) ? `${separator}${key}=[redacted]` : match;
+    });
+}
+/** The label `role=button[name="Save"]` wants: quotes and backslashes escaped. */
+function quoted(value) {
+    return `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
+}
+// Plain JavaScript, run in each frame: the project compiles without the DOM's types.
+const SCAN = `(limit) => {
+    const roles = { A: 'link', BUTTON: 'button', SELECT: 'combobox', TEXTAREA: 'textbox', SUMMARY: 'button' }
+    const inputRoles = { button: 'button', submit: 'button', reset: 'button', image: 'button', checkbox: 'checkbox', radio: 'radio', range: 'slider', search: 'searchbox' }
+    const clean = (s) => (s || '').replace(/\\s+/g, ' ').trim()
+    const body = document.body
+    const text = body ? body.innerText || '' : ''
+    const found = []
+    if (!body) return { text, elements: found }
+    const query = 'a[href], button, input:not([type=hidden]), select, textarea, summary, [role=button], [role=link], [role=checkbox], [role=radio], [role=tab], [role=menuitem], [role=option], [role=switch], [role=combobox], [role=textbox], [contenteditable=""], [contenteditable=true]'
+    for (const el of document.querySelectorAll(query)) {
+        if (found.length >= limit) break
+        const rect = el.getBoundingClientRect()
+        if (rect.width === 0 || rect.height === 0) continue
+        if (typeof el.checkVisibility === 'function' && !el.checkVisibility({ visibilityProperty: true, opacityProperty: true })) continue
+        const tag = el.tagName
+        const type = (el.getAttribute('type') || 'text').toLowerCase()
+        const role = el.getAttribute('role') || (tag === 'INPUT' ? inputRoles[type] || 'textbox' : roles[tag]) || (el.isContentEditable ? 'textbox' : tag.toLowerCase())
+        let name = clean(el.getAttribute('aria-label'))
+        const labelledBy = el.getAttribute('aria-labelledby')
+        if (!name && labelledBy) name = clean(labelledBy.split(/\\s+/).map((id) => document.getElementById(id)?.innerText || '').join(' '))
+        if (!name && el.labels && el.labels.length) name = clean([...el.labels].map((l) => l.innerText).join(' '))
+        if (!name && tag !== 'INPUT' && tag !== 'TEXTAREA' && tag !== 'SELECT') name = clean(el.innerText)
+        if (!name && tag === 'INPUT' && ['button', 'submit', 'reset'].includes(type)) name = clean(el.value)
+        if (!name) name = clean(el.getAttribute('placeholder') || el.getAttribute('title') || el.getAttribute('alt'))
+        found.push({ role, name, id: el.id || null, nameAttr: el.getAttribute('name'), tag: tag.toLowerCase(), x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 })
+    }
+    return { text, elements: found }
+}`;
+function selectorFor(element) {
+    if (element.id && /^[A-Za-z][\w-]*$/.test(element.id))
+        return `#${element.id}`;
+    if (element.nameAttr)
+        return `${element.tag}[name=${quoted(element.nameAttr)}]`;
+    if (element.name)
+        return `role=${element.role}[name=${quoted(element.name)}]`;
+    return element.tag;
+}
+/** `label=Email` is the field a label names, as Sifur writes the sign-in for a project. */
+export function locate(frame, selector) {
+    const label = /^label=(.+)$/.exec(selector)?.[1];
+    return (label ? frame.getByLabel(label, { exact: true }) : frame.locator(selector)).first();
+}
+/** The center of what the step names, in the top page's coordinates, when it's on the screen. */
+async function targetCenter(page, step) {
+    if (!step.selector)
+        return null;
+    for (const frame of page.frames()) {
+        try {
+            const box = await locate(frame, step.selector).boundingBox({ timeout: 500 });
+            if (box)
+                return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
+        }
+        catch { }
+    }
+    return null;
+}
+/** Where a frame's own coordinates start on the top page. */
+async function frameOffset(frame, page) {
+    if (frame === page.mainFrame())
+        return { x: 0, y: 0 };
+    const box = await (await frame.frameElement()).boundingBox();
+    return box ? { x: box.x, y: box.y } : { x: 0, y: 0 };
+}
+async function capture(page, step, secrets) {
+    const texts = [];
+    const elements = [];
+    let framesUnavailable = 0;
+    for (const frame of page.frames()) {
+        try {
+            const scan = await frame.evaluate(`(${SCAN})(${MAX_ELEMENTS * 10})`);
+            const offset = await frameOffset(frame, page).catch(() => ({ x: 0, y: 0 }));
+            texts.push(scan.text);
+            for (const element of scan.elements) {
+                elements.push({
+                    role: element.role,
+                    name: redact(element.name, secrets).slice(0, MAX_NAME),
+                    selector: redact(selectorFor(element), secrets).slice(0, MAX_SELECTOR),
+                    x: element.x + offset.x,
+                    y: element.y + offset.y,
+                });
+            }
+        }
+        catch {
+            // The top page unread is no evidence at all; a frame unread is said, not guessed at.
+            if (frame === page.mainFrame())
+                return null;
+            framesUnavailable++;
+        }
+    }
+    const target = await targetCenter(page, step);
+    if (target) {
+        const distance = (e) => Math.hypot(e.x - target.x, e.y - target.y);
+        elements.sort((a, b) => distance(a) - distance(b));
+    }
+    return {
+        url: redactUrl(page.url(), secrets).slice(0, MAX_URL),
+        text: redact(texts.join('\n').replace(/\s+/g, ' ').trim(), secrets).slice(0, MAX_TEXT),
+        elements: elements.slice(0, MAX_ELEMENTS).map(({ role, name, selector }) => ({ role, name, selector })),
+        ...(framesUnavailable > 0 ? { framesUnavailable } : {}),
+    };
+}
+/**
+ * What the screen showed when a step failed, for telling a wrong app from a wrong test. Null rather
+ * than an error when it can't be read: the result matters more than the evidence.
+ */
+export async function captureEvidence(page, step, secrets) {
+    let timer;
+    try {
+        return await Promise.race([
+            capture(page, step, secrets),
+            new Promise((resolve) => {
+                timer = setTimeout(() => resolve(null), CAPTURE_TIMEOUT);
+            }),
+        ]);
+    }
+    catch {
+        return null;
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}

package/dist/record.js

@@ -0,0 +1,333 @@
+import { spawn } from 'node:child_process';
+import { readFileSync, rmSync } from 'node:fs';
+import { mkdtemp, readFile, writeFile } from 'node:fs/promises';
+import { createRequire } from 'node:module';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { download, parseBaseUrl, ticketLink } from './accept.js';
+import { ApiError, CliError } from './api.js';
+import { launchChannel } from './browser.js';
+import { parseRecording, SUPPORTED } from './codegen.js';
+import { describe, perform } from './runner.js';
+import { collectSecrets, newRunToken, redact, resolveStep, RUN, secretUses } from './secrets.js';
+const ASSERTIONS = new Set(['expectText', 'expectVisible', 'expectUrl']);
+const PLACEHOLDER_ONLY = /^\{\{[A-Z][A-Z0-9_]*\}\}$/;
+const PLACEHOLDER_NAME = /^[A-Z][A-Z0-9_]*$/;
+// A field a sign-in typically has, by its selector or label: what's typed there is a credential.
+const CREDENTIAL_FIELD = /pass(word|code)?\b|passwd|pwd|secret|token|e-?mail|user[\s_-]?name|login|type=["']?password/i;
+const MAX_STEPS = 200;
+// As the runner plays the case, so the recording sees the same layout.
+const VIEWPORT = { width: 1440, height: 900 };
+const TIMEOUT = 60_000;
+/**
+ * The sign-in Sifur writes in front of a UI case: a goto, fills that each type one placeholder,
+ * then the click on the submit button. Matched by that shape rather than as the steps every case
+ * shares, which with one case, or cases that all open the same screen, would take in more.
+ */
+export function signInPrefix(script) {
+    const [open, ...rest] = script;
+    if (open?.action !== 'goto' || !open.url)
+        return [];
+    let fills = 0;
+    while (rest[fills]?.action === 'fill' && PLACEHOLDER_ONLY.test(String(rest[fills].value ?? '')))
+        fills++;
+    const submit = rest[fills];
+    if (fills === 0 || submit?.action !== 'click' || !/^role=button\[name=".*"\]$/.test(submit.selector ?? ''))
+        return [];
+    return script.slice(0, fills + 2);
+}
+/** The sign-in of the case being recorded, else of any UI case: it's the same for every one. */
+export function findSignIn(cases, target) {
+    const candidates = [target, ...cases.filter((c) => c.check !== 'automated')];
+    for (const testCase of candidates) {
+        const prefix = signInPrefix(testCase?.uiScript ?? []);
+        if (prefix.length > 0)
+            return prefix;
+    }
+    return [];
+}
+async function playSignIn(prefix, options) {
+    const { browser, channel } = await launchChannel({ headed: options.headed });
+    try {
+        const context = await browser.newContext({ viewport: VIEWPORT, colorScheme: 'light' });
+        const page = await context.newPage();
+        page.setDefaultTimeout(options.timeoutMs);
+        const runToken = newRunToken();
+        let signInUrl = '';
+        for (const [index, step] of prefix.entries()) {
+            try {
+                await perform(page, resolveStep(step, options.secrets, runToken), options.baseUrl, options.timeoutMs);
+            }
+            catch (error) {
+                const message = redact(String(error?.message ?? error).split('\n')[0] ?? '', options.secrets);
+                throw new CliError(`Couldn't sign in: step ${index + 1} (${step.caption ?? describe(step)}) failed: ${message}`);
+            }
+            if (index === 0)
+                signInUrl = page.url();
+        }
+        let stayed = false;
+        if (prefix.length > 0) {
+            // The session is there once the browser has left the sign-in page, not when the button is clicked.
+            await page.waitForURL((url) => url.toString() !== signInUrl, { timeout: Math.min(options.timeoutMs, 15_000) }).catch(() => {
+                stayed = true;
+            });
+            await page.waitForLoadState('load').catch(() => { });
+        }
+        const url = prefix.length > 0 ? page.url() : options.baseUrl;
+        return { storageState: await context.storageState({ indexedDB: true }), url, channel, stayed };
+    }
+    finally {
+        await browser.close().catch(() => { });
+    }
+}
+/** Playwright's documented recorder, from the playwright-core this CLI ships with (not whatever npx finds). */
+export function codegenCommand(options) {
+    const require = createRequire(import.meta.url);
+    const manifest = require.resolve('playwright-core/package.json');
+    const bin = JSON.parse(readFileSync(manifest, 'utf8')).bin['playwright-core'];
+    return {
+        command: process.execPath,
+        args: [
+            join(dirname(manifest), bin),
+            'codegen',
+            '--target',
+            'playwright-test',
+            '--channel',
+            options.channel,
+            '--load-storage',
+            options.storageState,
+            '--viewport-size',
+            `${VIEWPORT.width},${VIEWPORT.height}`,
+            '--color-scheme',
+            'light',
+            '-o',
+            options.output,
+            options.url,
+        ],
+    };
+}
+async function runCodegen(options) {
+    const { command, args } = codegenCommand(options);
+    await new Promise((resolve, reject) => {
+        // No stdin: the CLI's own questions keep the terminal.
+        const child = spawn(command, args, { stdio: ['ignore', 'inherit', 'inherit'] });
+        child.on('error', (error) => reject(new CliError(`Couldn't start Playwright's recorder: ${error.message}`)));
+        child.on('exit', () => resolve());
+    });
+}
+/** Removes the folder on Ctrl-C too: it holds the signed-in session, and may hold what was typed. */
+function removeOnExit(dir) {
+    const remove = () => rmSync(dir, { recursive: true, force: true });
+    const onSignal = (signal) => {
+        remove();
+        process.exit(signal === 'SIGINT' ? 130 : 143);
+    };
+    process.on('exit', remove);
+    process.on('SIGINT', onSignal);
+    process.on('SIGTERM', onSignal);
+    return () => {
+        process.off('exit', remove);
+        process.off('SIGINT', onSignal);
+        process.off('SIGTERM', onSignal);
+        remove();
+    };
+}
+/**
+ * Signs in, records, and reads what the recorder wrote. The signed-in session and the raw recording
+ * (which can hold anything typed, passwords included) live in a private temporary folder that is
+ * removed before this returns, whatever happens.
+ */
+export async function record(prefix, options, deps) {
+    const { io } = deps;
+    const dir = await mkdtemp(join(tmpdir(), 'sifur-record-'));
+    const cleanup = removeOnExit(dir);
+    try {
+        if (prefix.length > 0)
+            io.out('Signing in…');
+        const signedIn = await (deps.signIn ?? playSignIn)(prefix, options);
+        if (signedIn.stayed)
+            io.out("Note: the browser is still on the sign-in page after signing in. If the recorder opens there, the sign-in didn't work.");
+        const storageState = join(dir, 'state.json');
+        await writeFile(storageState, JSON.stringify(signedIn.storageState), { mode: 0o600 });
+        const output = join(dir, 'recording.spec.ts');
+        io.out(`Recording in a new ${signedIn.channel === 'msedge' ? 'Edge' : 'Chrome'} window, starting at ${signedIn.url}${prefix.length > 0 ? ', signed in' : ''}.`);
+        io.out('Do the case\'s steps, then check the result with the recorder toolbar\'s "Assert text" or "Assert visibility".');
+        io.out('Close the window when you\'re done.');
+        await (deps.codegen ?? runCodegen)({ url: signedIn.url, storageState, output, channel: signedIn.channel });
+        let source;
+        try {
+            source = await readFile(output, 'utf8');
+        }
+        catch {
+            throw new CliError('Nothing was recorded: the recorder closed without writing a test.');
+        }
+        return { source, startUrl: signedIn.url };
+    }
+    finally {
+        cleanup();
+    }
+}
+/** The secrets the engineer has given this CLI: the sign-in's, and every SIFUR_SECRET_<NAME>. */
+export function knownSecrets(signIn, env) {
+    const known = {};
+    for (const [key, value] of Object.entries(env)) {
+        const name = /^SIFUR_SECRET_([A-Z][A-Z0-9_]*)$/.exec(key)?.[1];
+        if (name && value)
+            known[name] = value;
+    }
+    return { ...known, ...signIn };
+}
+async function askPlaceholder(io, step, selector) {
+    io.out(`Step ${step} types into ${selector}, which looks like a sign-in field. What's typed there is never uploaded:`);
+    for (;;) {
+        const name = (await io.ask('  Name a placeholder for it, like APP_EMAIL or APP_PASSWORD: ')).trim().replace(/^\{\{|\}\}$/g, '').toUpperCase();
+        if (PLACEHOLDER_NAME.test(name) && name !== RUN)
+            return name;
+        io.out('  Use capital letters, digits and underscores, starting with a letter (and not RUN).');
+    }
+}
+/**
+ * Credentials out of the steps: a known secret's value becomes its `{{NAME}}`, and anything else
+ * typed into a password, email or username field becomes a placeholder the engineer names.
+ */
+export async function withPlaceholders(steps, known, io) {
+    const secrets = Object.entries(known)
+        .filter(([, value]) => value !== '')
+        .sort(([, a], [, b]) => b.length - a.length);
+    // A short value inside a longer one is more likely a coincidence than the secret.
+    const replaceIn = (text) => {
+        let replaced = text;
+        for (const [name, value] of secrets)
+            if (value.length >= 4)
+                replaced = replaced.replaceAll(value, `{{${name}}}`);
+        return replaced;
+    };
+    const asked = new Map();
+    const named = [];
+    const result = [];
+    for (const [index, original] of steps.entries()) {
+        const step = { ...original };
+        if (step.action === 'fill' && typeof step.value === 'string' && step.value !== '') {
+            const exact = secrets.find(([, value]) => value === step.value);
+            step.value = exact ? `{{${exact[0]}}}` : replaceIn(step.value);
+            if (!/\{\{[A-Z][A-Z0-9_]*\}\}/.test(step.value) && CREDENTIAL_FIELD.test(step.selector ?? '')) {
+                let name = asked.get(step.value);
+                if (!name) {
+                    name = await askPlaceholder(io, index + 1, step.selector ?? 'a field');
+                    asked.set(step.value, name);
+                    named.push(name);
+                }
+                step.value = `{{${name}}}`;
+            }
+        }
+        if (step.action === 'goto' && step.url)
+            step.url = replaceIn(step.url);
+        if ((step.action === 'expectText' || step.action === 'expectUrl') && step.text)
+            step.text = replaceIn(step.text);
+        result.push(step);
+    }
+    return { steps: result, named };
+}
+export function formatStep(step) {
+    const target = step.selector ?? step.url ?? '';
+    const detail = step.value !== undefined ? `  ${step.value}` : step.text !== undefined ? `  "${step.text}"` : '';
+    return `${step.action} ${target}${detail}`.trim();
+}
+async function confirmUpload(io, count) {
+    const answer = (await io.ask(`Upload these ${count} step${count === 1 ? '' : 's'}? [y/N] `)).trim().toLowerCase();
+    return answer === 'y' || answer === 'yes';
+}
+function findCase(cases, id) {
+    const found = cases.find((c) => c.id.toLowerCase() === id.toLowerCase() || c.uuid === id);
+    if (!found)
+        throw new CliError(`There's no ${id} in this ticket's cases: ${cases.map((c) => c.id).join(', ') || 'it has none'}.`);
+    return found;
+}
+/** The whole `record` command: 0 when the steps were uploaded, 1 when the engineer chose not to. Errors throw. */
+export async function recordCase(api, args, deps) {
+    const { io } = deps;
+    const env = deps.env ?? process.env;
+    if (!args.caseId === !args.title)
+        throw new CliError('Name the case to record again (TC-3), or use --new "<title>" for a new one.');
+    if (args.repository && !args.title)
+        throw new CliError('--repository goes with --new: a case keeps its repository.');
+    const saved = await download(api, args.ref);
+    const { acceptance } = saved;
+    const target = args.caseId ? findCase(acceptance.cases, args.caseId) : undefined;
+    if (target && typeof target.version !== 'number') {
+        throw new CliError(`This Sifur doesn't say which version ${target.id} is on, so a recording can't be checked against it. Sifur needs updating first.`);
+    }
+    const url = args.url ?? acceptance.appUrl ?? null;
+    if (!url) {
+        throw new CliError("--url <test url> is required: the test server the branch is deployed to. Or set it once in the project's test environment settings.");
+    }
+    const baseUrl = parseBaseUrl(url);
+    io.out(target ? `${acceptance.ticket.key} ${target.id}: ${target.title}` : `${acceptance.ticket.key}: a new case, "${args.title}"`);
+    if (!args.url)
+        io.out(`Recording on ${baseUrl} (the project's test server). Use --url for another.`);
+    const prefix = findSignIn(acceptance.cases, target);
+    const uses = secretUses([{ uuid: '', id: target?.id ?? 'The sign-in', title: '', check: 'ui', uiScript: prefix }]);
+    const signInSecrets = await collectSecrets(uses, io, env);
+    const missing = uses.filter((use) => signInSecrets[use.name] === undefined);
+    if (missing.length > 0)
+        throw new CliError(`Recording starts signed in, which needs ${missing.map((use) => use.name).join(' and ')}.`);
+    const timeoutMs = deps.timeoutMs ?? TIMEOUT;
+    const { source, startUrl } = await record(prefix, { baseUrl, secrets: signInSecrets, headed: args.headed, timeoutMs }, deps);
+    // Only a sign-in leaves the browser at the start page; without one, the case has to open it itself.
+    const parsed = parseRecording(source, { baseUrl, startUrl: prefix.length > 0 ? startUrl : undefined });
+    if (parsed.refused.length > 0) {
+        const lines = parsed.refused.map((r) => `  line ${r.line}: ${r.code}\n    ${r.reason}`);
+        throw new CliError(`Sifur can't replay ${parsed.refused.length === 1 ? 'one line' : `${parsed.refused.length} lines`} of the recording, so it wasn't uploaded:\n${lines.join('\n')}\n${SUPPORTED}`);
+    }
+    if (parsed.steps.length === 0)
+        throw new CliError('Nothing was recorded.');
+    if (!parsed.steps.some((step) => ASSERTIONS.has(step.action))) {
+        throw new CliError('The recording has no check, so nothing in it proves the case. Record it again, and after your steps use the recorder toolbar\'s "Assert text" or "Assert visibility" on what the change shows.');
+    }
+    if (parsed.steps.length > MAX_STEPS)
+        throw new CliError(`The recording has ${parsed.steps.length} steps; a case can have at most ${MAX_STEPS}.`);
+    const { steps, named } = await withPlaceholders(parsed.steps, knownSecrets(signInSecrets, env), io);
+    io.out();
+    io.out(`${steps.length} step${steps.length === 1 ? '' : 's'}${prefix.length > 0 ? ", after Sifur's sign-in" : ''}:`);
+    for (const [index, step] of steps.entries())
+        io.out(`  ${index + 1}. ${formatStep(step)}`);
+    if (named.length > 0)
+        io.out(`When the case runs, ${named.map((name) => `{{${name}}}`).join(' and ')} come${named.length === 1 ? 's' : ''} from SIFUR_SECRET_<NAME>, or are asked for.`);
+    if (!(await confirmUpload(io, steps.length))) {
+        io.out('Nothing was uploaded.');
+        return 1;
+    }
+    let result;
+    try {
+        result = target
+            ? await api.request(`tickets/${acceptance.ticket.uuid}/test-cases/${target.uuid}/steps`, {
+                method: 'PUT',
+                body: { stateVersion: acceptance.ticket.stateVersion, caseVersion: target.version, uiSteps: steps },
+                organization: saved.organizationUuid,
+            })
+            : await api.request(`tickets/${acceptance.ticket.uuid}/acceptance/cases`, {
+                method: 'POST',
+                body: {
+                    stateVersion: acceptance.ticket.stateVersion,
+                    title: args.title,
+                    ...(args.repository ? { repository: args.repository } : {}),
+                    uiSteps: steps,
+                },
+                organization: saved.organizationUuid,
+            });
+    }
+    catch (error) {
+        if (error instanceof ApiError && error.status === 409) {
+            throw new CliError(`Sifur refused the recording: ${error.message} Nothing was saved; run \`sifur record\` again.`);
+        }
+        if (error instanceof ApiError && error.status === 422)
+            throw new CliError(`Sifur refused the recording: ${error.message}`);
+        throw error;
+    }
+    io.out(target ? `Saved as version ${result.version} of ${result.id}.` : `Added ${result.id}.`);
+    io.out(`The acceptance run starts again with the cases as they are now: run \`sifur accept ${acceptance.ticket.key}\` when you're ready.`);
+    const link = ticketLink(args.consoleOrigin, acceptance.ticket.uuid);
+    if (link)
+        io.out(link);
+    return 0;
+}

package/dist/results.js

@@ -16,6 +16,8 @@ export function buildResultsPayload(stateVersion, baseUrl, results, screenshots)
                 screenshotUuid: step.screenshot ? (screenshots.get(step.screenshot) ?? null) : null,
             })),
             ...(result.output !== undefined ? { output: result.output } : {}),
+            ...(result.evidence ? { evidence: result.evidence } : {}),
+            ...(result.engineerVerdict ? { engineerVerdict: result.engineerVerdict } : {}),
         })),
     };
 }

package/dist/runner.js

@@ -1,5 +1,6 @@
 import { mkdir } from 'node:fs/promises';
 import { join } from 'node:path';
+import { captureEvidence, locate } from './evidence.js';
 import { newRunToken, redact, resolveStep } from './secrets.js';
 const ASSERTIONS = new Set(['expectText', 'expectVisible', 'expectUrl']);
 const TIMEOUT = 60_000;
@@ -27,7 +28,7 @@ async function find(page, selector, timeoutMs) {
         throw new Error('the step has no selector');
     return eventually(async () => {
         for (const frame of page.frames()) {
-            const match = frame.locator(selector).first();
+            const match = locate(frame, selector);
             if ((await match.count()) > 0 && (await match.isVisible().catch(() => false)))
                 return match;
         }
@@ -47,6 +48,10 @@ async function screenText(page) {
 export function caseUrl(path, baseUrl) {
     if (/^[a-z][a-z0-9+.-]*:/i.test(path))
         return path;
+    return new URL(path.replace(/^\/+/, ''), appFolder(baseUrl)).toString();
+}
+/** The folder a case's paths open under. */
+export function appFolder(baseUrl) {
     const base = new URL(baseUrl);
     base.search = '';
     base.hash = '';
@@ -56,9 +61,9 @@ export function caseUrl(path, baseUrl) {
             ? base.pathname.replace(/[^/]*$/, '')
             : `${base.pathname}/`;
     }
-    return new URL(path.replace(/^\/+/, ''), base).toString();
+    return base;
 }
-async function perform(page, step, baseUrl, settleMs) {
+export async function perform(page, step, baseUrl, settleMs) {
     const url = step.url ? caseUrl(step.url, baseUrl) : null;
     switch (step.action) {
         case 'goto':
@@ -148,6 +153,7 @@ export async function runUiCase(browser, testCase, options) {
     const steps = [];
     const notes = [];
     let status = 'passed';
+    let evidence = null;
     try {
         for (const [index, step] of (testCase.uiScript ?? []).entries()) {
             const n = index + 1;
@@ -155,12 +161,14 @@ export async function runUiCase(browser, testCase, options) {
             let ok = true;
             let error = null;
             options.onStep?.(n, caption);
+            const resolved = resolveStep(step, secrets, runToken);
             try {
-                await perform(page, resolveStep(step, secrets, runToken), options.baseUrl, options.timeoutMs ?? TIMEOUT);
+                await perform(page, resolved, options.baseUrl, options.timeoutMs ?? TIMEOUT);
             }
             catch (caught) {
                 ok = false;
                 error = redact(String(caught?.message ?? caught).split('\n')[0] ?? '', secrets);
+                evidence = await captureEvidence(page, resolved, secrets);
             }
             let screenshot = join(dir, `${String(n).padStart(2, '0')}.png`);
             await page.screenshot({ path: screenshot, fullPage: true }).catch(() => {
@@ -174,6 +182,7 @@ export async function runUiCase(browser, testCase, options) {
                     notes.push(`Step ${n} (${caption}): ${answer.note}`);
                 if (!answer.ok) {
                     ok = false;
+                    evidence = await captureEvidence(page, resolved, secrets);
                     error = answer.note ? `The engineer judged it wrong: ${answer.note}` : 'The engineer judged it wrong.';
                 }
             }
@@ -191,5 +200,5 @@ export async function runUiCase(browser, testCase, options) {
     finally {
         await context.close().catch(() => { });
     }
-    return { status, steps, notes };
+    return { status, steps, notes, ...(evidence ? { evidence } : {}) };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sifurai/cli",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "Run a Sifur ticket's acceptance cases against your own test server and upload the results.",
   "license": "UNLICENSED",
   "homepage": "https://sifur.ai",