jevway 0.1.2 → 0.3.0

package/dist/inbox/inbox.js

@@ -0,0 +1,127 @@
+import { randomBytes } from 'node:crypto';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { callApi, cancelled, InboxError, membersOf } from './mail-api.js';
+import { bodyText, extractCode, extractLinks, matchesFilters } from './message.js';
+export { InboxError } from './mail-api.js';
+/** The mail.tm-compatible providers, mail.tm first; the tools take a name, never a URL. */
+export const PROVIDERS = { 'mail.tm': 'https://api.mail.tm', 'mail.gw': 'https://api.mail.gw' };
+const POLL_INTERVAL_MS = 3_000;
+const FAILED_POLLS_LIMIT = 3;
+/** An inbox from stored credentials, with the password hidden from serialization. */
+export const inboxFrom = ({ address, password, api }) => Object.defineProperty({ address, api }, 'password', { value: password, enumerable: false });
+/** `mail.tm` for its API URL; the URL itself for any other. */
+export const providerOf = (api) => Object.keys(PROVIDERS).find((name) => PROVIDERS[name] === api) ?? api;
+const summaryOf = (message) => ({
+    id: message.id ?? '',
+    from: message.from?.address ?? '',
+    subject: message.subject ?? '',
+    received_at: message.createdAt ?? '',
+});
+const receivedOf = (message) => {
+    const code = extractCode(message);
+    return { ...summaryOf(message), ...(code && { code }), links: extractLinks(message), text: bodyText(message) };
+};
+const createOn = async (api, client) => {
+    const domains = membersOf(await callApi(client, api, '/domains'));
+    const domain = domains.find((entry) => entry.isActive !== false)?.domain;
+    if (!domain)
+        throw new InboxError('provider_rejected', `${api} offers no active domain`, 'Try the other provider.');
+    const address = `test-${randomBytes(5).toString('hex')}@${domain}`;
+    const password = randomBytes(18).toString('base64url');
+    await callApi(client, api, '/accounts', { method: 'POST', body: { address, password }, secrets: [password] });
+    return inboxFrom({ address, password, api });
+};
+/**
+ * A new inbox on `api`, or on mail.tm and then, if mail.tm is down, offers no active domain or refuses the account,
+ * on mail.gw once. Both failing is `inbox_unavailable`.
+ *
+ * @example
+ * const inbox = await createInbox(); // { address: 'test-0a1b2c3d4e@<domain>', api: 'https://api.mail.tm' }
+ */
+export const createInbox = async ({ api, ...client } = {}) => {
+    if (api)
+        return createOn(api, client);
+    try {
+        return await createOn(PROVIDERS['mail.tm'], client);
+    }
+    catch (first) {
+        if (!(first instanceof InboxError))
+            throw first;
+        try {
+            return await createOn(PROVIDERS['mail.gw'], client);
+        }
+        catch (second) {
+            throw new InboxError('inbox_unavailable', `No inbox could be made: mail.tm failed (${first.message}), then mail.gw failed (${second.message}).`, 'Both disposable inbox providers are down or refusing new accounts. Ask the user for a test mailbox of their own.');
+        }
+    }
+};
+const fetchToken = async (inbox, client, signal) => (await callApi(client, inbox.api, '/token', {
+    method: 'POST',
+    body: { address: inbox.address, password: inbox.password },
+    secrets: [inbox.password],
+    signal,
+})).token;
+const listWith = async (inbox, client, token, signal) => membersOf(await callApi(client, inbox.api, '/messages', { token, signal, secrets: [inbox.password] }));
+const readWith = async (inbox, client, token, id, signal) => {
+    try {
+        return receivedOf(await callApi(client, inbox.api, `/messages/${encodeURIComponent(id)}`, { token, signal, secrets: [inbox.password] }));
+    }
+    catch (error) {
+        if (error instanceof InboxError && error.status === 404) {
+            throw new InboxError('message_not_found', `No message "${id}" in ${inbox.address}`, 'List the inbox for its message ids.', 404);
+        }
+        throw error;
+    }
+};
+/** The inbox's messages, newest first, without their bodies. */
+export const listMessages = async (inbox, { signal, ...client } = {}) => (await listWith(inbox, client, await fetchToken(inbox, client, signal), signal)).map(summaryOf);
+/** One message in full, with its code and links. */
+export const readMessage = async (inbox, id, { signal, ...client } = {}) => readWith(inbox, client, await fetchToken(inbox, client, signal), id, signal);
+const defaultDelay = async (ms, signal) => {
+    try {
+        await sleep(ms, undefined, { signal });
+    }
+    catch {
+        throw cancelled();
+    }
+};
+/**
+ * Polls the inbox now and every 3 s until the newest message matching `from` and `subject` arrives, then reads it
+ * in full; undefined once `timeoutMs` has passed (with one last poll at the deadline). A 401 gets a new token; 3
+ * failed polls in a row end the wait with the provider's error. Aborting `signal` stops it with `cancelled`.
+ *
+ * @example
+ * await waitForMessage(inbox, { from: 'staylist', timeoutMs: 120_000 }) // { code: '482913', links: [...], … }
+ */
+export const waitForMessage = async (inbox, { from, subject, timeoutMs, signal, onPoll, ...client }) => {
+    const now = client.now ?? Date.now;
+    const delay = client.delay ?? defaultDelay;
+    const started = now();
+    let token = await fetchToken(inbox, client, signal);
+    let failures = 0;
+    for (;;) {
+        if (signal?.aborted)
+            throw cancelled();
+        onPoll?.(now() - started);
+        try {
+            const match = (await listWith(inbox, client, token, signal)).find((message) => matchesFilters(message, { from, subject }));
+            if (match?.id)
+                return await readWith(inbox, client, token, match.id, signal);
+            failures = 0;
+        }
+        catch (error) {
+            if (!(error instanceof InboxError) || error.code === 'cancelled' || error.code === 'message_not_found')
+                throw error;
+            if (++failures >= FAILED_POLLS_LIMIT)
+                throw error;
+            if (error.status === 401) {
+                token = await fetchToken(inbox, client, signal);
+                continue;
+            }
+        }
+        const remaining = started + timeoutMs - now();
+        if (remaining <= 0)
+            return undefined;
+        await delay(Math.min(POLL_INTERVAL_MS, remaining), signal);
+    }
+};

package/dist/inbox/mail-api.js

@@ -0,0 +1,52 @@
+/**
+ * A failed inbox call, with what to do in `hint`. Codes: `provider_unavailable` (network, 5xx, 429),
+ * `provider_rejected` (another 4xx), `inbox_unavailable` (both providers failed), `unknown_inbox`,
+ * `message_not_found`, `cancelled`.
+ */
+export class InboxError extends Error {
+    code;
+    hint;
+    status;
+    constructor(code, message, hint, status) {
+        super(message);
+        this.code = code;
+        this.hint = hint;
+        this.status = status;
+    }
+}
+const RETRY_HINT = 'The provider is busy or down; try again in a minute, or ask the user for a test mailbox of their own.';
+const scrub = (text, secrets) => secrets.filter(Boolean).reduce((clean, secret) => clean.split(secret).join('[secret]'), text);
+export const cancelled = () => new InboxError('cancelled', 'Stopped waiting: cancelled by the caller.', 'Wait again when you need the email.');
+/**
+ * One call to a mail.tm-compatible API, JSON in and out. Failures are `InboxError`s whose text never holds `secrets`
+ * (the password and token), even when the provider echoes them.
+ */
+export const callApi = async (client, api, path, { method = 'GET', body, token, signal, secrets = [] } = {}) => {
+    const fetch = client.fetch ?? globalThis.fetch;
+    const where = `${method} ${api}${path}`;
+    let response;
+    try {
+        response = await fetch(`${api}${path}`, {
+            method,
+            headers: {
+                accept: 'application/ld+json',
+                ...(body && { 'content-type': 'application/json' }),
+                ...(token && { authorization: `Bearer ${token}` }),
+            },
+            body: body && JSON.stringify(body),
+            signal,
+        });
+    }
+    catch (error) {
+        if (signal?.aborted)
+            throw cancelled();
+        throw new InboxError('provider_unavailable', `${where} failed: ${scrub(error.message, [...secrets, token ?? ''])}`, RETRY_HINT);
+    }
+    if (response.ok)
+        return (await response.json());
+    const detail = scrub((await response.text().catch(() => '')).slice(0, 300), [...secrets, token ?? '']);
+    const transient = response.status >= 500 || response.status === 429;
+    throw new InboxError(transient ? 'provider_unavailable' : 'provider_rejected', `${where} → ${response.status} ${detail}`.trim(), transient ? RETRY_HINT : 'The provider refused the request; the message says why.', response.status);
+};
+/** A collection as `{ 'hydra:member': [...] }` (ld+json) or a bare array (plain JSON). */
+export const membersOf = (collection) => Array.isArray(collection) ? collection : (collection['hydra:member'] ?? []);

package/dist/inbox/message.js

@@ -0,0 +1,34 @@
+const EXCERPT_CHARS = 600;
+const htmlOf = (message) => [message.html ?? []].flat().join('\n');
+const stripHtml = (html) => html
+    .replace(/<(style|script)[\s\S]*?<\/\1>/gi, '')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+/** The text part, or the html part without tags when there is no text. */
+export const bodyText = (message) => message.text || stripHtml(htmlOf(message));
+/**
+ * The first standalone 4–8 digit number in subject and body: not part of a longer number, a decimal, a date or an id.
+ *
+ * @example
+ * extractCode({ subject: 'Welcome', text: 'Your code is 482913.' }) // '482913'
+ */
+export const extractCode = (message) => `${message.subject ?? ''}\n${bodyText(message)}`.match(/(?<![\w.,/-])\d{4,8}(?!\d|[.,/-]\d)/)?.[0];
+// Doctype and namespace URLs and images (logos, tracking pixels) are never the link to follow.
+const NOT_A_LINK = /^https?:\/\/www\.w3\.org\/|\.(png|jpe?g|gif|svg|webp|ico)(\?|$)/i;
+/** Unique http(s) URLs a person could follow, from the text and html parts, `&amp;` decoded. */
+export const extractLinks = (message) => {
+    const urls = `${message.text ?? ''}\n${htmlOf(message)}`.match(/https?:\/\/[^\s"'<>)\]]+/g) ?? [];
+    return [...new Set(urls.map((url) => url.replace(/&amp;/g, '&')))].filter((url) => !NOT_A_LINK.test(url));
+};
+/** Whether the sender (name or address) and subject contain the filters, ignoring case; a missing filter passes. */
+export const matchesFilters = (message, { from, subject }) => {
+    const sender = `${message.from?.name ?? ''} ${message.from?.address ?? ''}`.toLowerCase();
+    return ((!from || sender.includes(from.toLowerCase())) &&
+        (!subject || (message.subject ?? '').toLowerCase().includes(subject.toLowerCase())));
+};
+/** `text` with whitespace folded, cut at 600 characters with an ellipsis. */
+export const excerpt = (text) => {
+    const folded = text.replace(/\s+/g, ' ').trim();
+    return folded.length <= EXCERPT_CHARS ? folded : `${folded.slice(0, EXCERPT_CHARS).trimEnd()}…`;
+};

package/dist/jev/replay.js

@@ -15,7 +15,6 @@ export class ReplayMiss extends Error {
         this.pageDiff = pageDiff;
     }
 }
-const DEFAULT_DIR = 'verify/out/replay/default';
 const IMAGE = '<image>';
 const LOCAL_ORIGIN = /\b(127\.0\.0\.1|localhost):\d+/g;
 /**
@@ -33,8 +32,13 @@ export const replayMode = () => {
         throw new Error(`JEVWAY_REPLAY must be record or replay, got "${mode}".`);
     return mode;
 };
-/** The recording folder: `JEVWAY_REPLAY_DIR`, or `verify/out/replay/default` under the working directory. */
-export const replayDir = () => resolve(process.env.JEVWAY_REPLAY_DIR?.trim() || DEFAULT_DIR);
+/** The recording folder, `JEVWAY_REPLAY_DIR`: required whenever `JEVWAY_REPLAY` is set, so no run writes into a guessed folder. */
+export const replayDir = () => {
+    const dir = process.env.JEVWAY_REPLAY_DIR?.trim();
+    if (!dir)
+        throw new Error('JEVWAY_REPLAY needs JEVWAY_REPLAY_DIR: the folder that holds the recordings.');
+    return resolve(dir);
+};
 const isPlainObject = (value) => typeof value === 'object' && value !== null && Object.getPrototypeOf(value) === Object.prototype;
 /**
  * The request as it is matched: keys sorted at every depth, every image (an SDK `Image`, an `image_url` part, a

package/dist/jev/retry.js

@@ -0,0 +1,29 @@
+/** A busy or flaky model API (5xx incl. 529 overload, 429, 499, timeouts, dropped connections): worth another try. */
+const TRANSIENT_MODEL_ERROR = /^(5\d\d|429|499)\b|timeout|timed out|ECONNRESET|ETIMEDOUT|fetch failed|socket hang up/i;
+/**
+ * Whether a model call failed for a reason that may pass on its own.
+ *
+ * @example
+ * isTransientModelError(new Error('529 We are currently experiencing high traffic')) // true
+ */
+export const isTransientModelError = (error) => error instanceof Error && TRANSIENT_MODEL_ERROR.test(error.message);
+/**
+ * An `Ask` that tries again after a transient failure, waiting each of `delaysMs` in turn; any other failure, or the
+ * last one, is thrown as it came.
+ *
+ * @example
+ * const ask = retryingAsk(createAsk(profile), { delaysMs: [1000, 3000] });
+ */
+export const retryingAsk = (ask, { delaysMs, wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) }) => async (request) => {
+    for (const delay of delaysMs) {
+        try {
+            return await ask(request);
+        }
+        catch (error) {
+            if (!isTransientModelError(error))
+                throw error;
+            await wait(delay);
+        }
+    }
+    return ask(request);
+};

package/dist/jev/secrets.js

@@ -29,15 +29,16 @@ const isImagePart = (value) => value.type === 'image_url';
  * caller marked secret never reaches the model or any output, including when the page echoes it (upper-cased, in an
  * address). Matching is case-insensitive and takes the longest form first, in one pass, so a marker is never scrubbed
  * again. Every non-empty value counts, even a short one: a garbled text is the safe failure, a leaked secret is not.
- * With nothing marked it is the identity.
+ * `more` adds secrets that are not supplied values, each under the key it stands for (an authenticator secret's
+ * base32, every code typed from it). With nothing marked it is the identity.
  *
  * @example
  * const scrub = scrubberFor({ token: 'ghp_x1' }, ['token']);
  * scrub({ page_text: 'Token GHP_X1 created' }) // { page_text: 'Token [secret:token] created' }
  */
-export const scrubberFor = (values, secretKeys = []) => {
+export const scrubberFor = (values, secretKeys = [], more = []) => {
     const keyByForm = new Map();
-    for (const [key, value] of secretEntries(values, secretKeys)) {
+    for (const [key, value] of [...secretEntries(values, secretKeys), ...more.filter(([, value]) => value.trim() !== '')]) {
         for (const form of formsOf(value))
             if (form && !keyByForm.has(form.toLowerCase()))
                 keyByForm.set(form.toLowerCase(), key);

package/dist/journeys/journey-doc.js

@@ -44,7 +44,8 @@ export const buildFlows = (steps) => {
     return [...flows.values()];
 };
 const parentOf = (name) => (name.includes('/') ? name.slice(0, name.lastIndexOf('/')) : null);
-const describeAction = (step) => {
+/** A step's action as a person reads it: `Tap "Search"`, `Type into "Destination" → "Lisbon"`. */
+export const describeAction = (step) => {
     const verb = { CLICK: 'Tap', TYPE_TEXT: 'Type into', SELECT: 'Choose' }[step.action.operation] ?? step.action.operation;
     return `${verb} "${step.action.label}"${step.action.value ? ` → "${step.action.value}"` : ''}`;
 };

package/dist/journeys/journey-session.js

@@ -10,6 +10,9 @@ import { buildFlows, hotspotOf } from './journey-doc.js';
 import { pickNextMove } from './pick-next-move.js';
 import { renderViewer } from './render-viewer.js';
 import { qaFindings, readScreenJudgments, screenQuestions, } from './screen-judgments.js';
+import { screenFileName } from '../output/layout.js';
+import { writeManifest } from '../output/manifest.js';
+import { outsideHosts } from '../task/hosts.js';
 const EXPLORE_GOAL = `Explore this site the way a first-time mobile visitor would, one screen at a time.
 Open the next screen that matters most for the site's core user journeys: discovering, searching, comparing,
 and evaluating its main offering. Prefer primary navigation, search, results, and detail screens over legal,
@@ -24,7 +27,7 @@ footer, social, app-download, or external links. Dismiss blocking banners. Never
  * journey.report();                                   // screen s01 + Jev's suggested next tap
  * await journey.step({ next: 4, flow: 'Search', insight: 'Search is a full-screen sheet' });
  * await journey.autopilot({ screens: 5, flow: 'Search/Results' });
- * journey.finish();                                   // → journey.json, index.html, screens/*.png
+ * journey.finish();                                   // → journey.json, index.html, manifest.json, screens/s01-home.png …
  */
 export class JourneySession {
     doc;
@@ -227,7 +230,8 @@ export class JourneySession {
                 continue;
             }
             const scope = this.#options.stayWithin ?? new URL(this.#options.startUrl).origin;
-            if (!(await this.#tab.observe()).url.startsWith(scope)) {
+            const landed = (await this.#tab.observe()).url;
+            if (!landed.startsWith(scope) || this.#outsideHosts(landed)) {
                 await this.#goBack();
                 this.#logMove('left_scope', move.action.label);
                 continue;
@@ -286,9 +290,15 @@ export class JourneySession {
         return this.report();
     }
     // Observe → one Jev request (next step + screen understanding) → dedupe → screenshot → QA → save.
+    #outsideHosts(url) {
+        return this.#options.hosts && outsideHosts(url, this.#options.hosts);
+    }
     async #capture(arrival) {
         await this.#diagnostics.waitForIdle();
         const page = await this.#settledPage();
+        const outside = this.#outsideHosts(page.url);
+        if (outside)
+            throw new Error(`The page is outside the server's hosts: ${outside}. Take \`back\` to return.`);
         const seesScreen = (this.#options.screenshots ?? 'off') !== 'off';
         const boxed = seesScreen && Boolean(this.#options.box);
         const decision = await decide(this.#ask, {
@@ -320,7 +330,7 @@ export class JourneySession {
             mergeFindings(this.#screen(existing).qa, findings);
         }
         else {
-            const screenshot = `screens/${id}.png`;
+            const screenshot = screenFileName(id, judgments.type);
             writeFileSync(join(this.outDir, screenshot), Buffer.from(await this.#tab.screenshot(), 'base64'));
             this.doc.screens.push({
                 id,
@@ -433,6 +443,7 @@ export class JourneySession {
         this.doc.flows = buildFlows(this.doc.steps);
         writeFileSync(join(this.outDir, 'journey.json'), JSON.stringify(this.doc, null, 2));
         writeFileSync(join(this.outDir, 'index.html'), renderViewer(this.doc));
+        writeManifest(this.outDir, 'journey');
     }
 }
 // "https://site/#/" and "https://site/" are the same route.