jevway 0.0.0-stage → 0.1.2

package/dist/jev/replay.js

@@ -0,0 +1,213 @@
+import { createHash } from 'node:crypto';
+import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { Image } from '@typesafe-ai/sdk';
+/**
+ * A request the recording never saw: the run can't be answered offline. The message names the request and the changed
+ * paths; the page-text lines are in `pageDiff` only, because the run retries a model error whose message reads like a
+ * timeout, and a page can say "timed out".
+ */
+export class ReplayMiss extends Error {
+    name = 'ReplayMiss';
+    pageDiff;
+    constructor(message, pageDiff = []) {
+        super(message);
+        this.pageDiff = pageDiff;
+    }
+}
+const DEFAULT_DIR = 'verify/out/replay/default';
+const IMAGE = '<image>';
+const LOCAL_ORIGIN = /\b(127\.0\.0\.1|localhost):\d+/g;
+/**
+ * `JEVWAY_REPLAY`: `record` saves every Jev answer next to its request, `replay` answers only from those recordings;
+ * unset means live. Any other value is a configuration error, never a silent live run.
+ *
+ * @example
+ * replayMode(); // 'replay' under JEVWAY_REPLAY=replay
+ */
+export const replayMode = () => {
+    const mode = process.env.JEVWAY_REPLAY?.trim();
+    if (!mode)
+        return undefined;
+    if (mode !== 'record' && mode !== 'replay')
+        throw new Error(`JEVWAY_REPLAY must be record or replay, got "${mode}".`);
+    return mode;
+};
+/** The recording folder: `JEVWAY_REPLAY_DIR`, or `verify/out/replay/default` under the working directory. */
+export const replayDir = () => resolve(process.env.JEVWAY_REPLAY_DIR?.trim() || DEFAULT_DIR);
+const isPlainObject = (value) => typeof value === 'object' && value !== null && Object.getPrototypeOf(value) === Object.prototype;
+/**
+ * The request as it is matched: keys sorted at every depth, every image (an SDK `Image`, an `image_url` part, a
+ * `data:image/…` URL) replaced by `<image>`, and every local fixture origin's port by `<port>`. Two runs of the same
+ * step on fresh fixture servers then read the same: their only differences are the random port and the screenshot's
+ * reference and pixels, and the screenshot is drawn from the same elements the text state lists.
+ *
+ * @example
+ * canonicalRequest({ state: { page: { url: 'http://127.0.0.1:63068/stays.html' } } });
+ * // { state: { page: { url: 'http://127.0.0.1:<port>/stays.html' } } }
+ */
+export const canonicalRequest = (value) => {
+    if (value instanceof Image || value instanceof Blob)
+        return IMAGE;
+    if (typeof value === 'string')
+        return value.startsWith('data:image/') ? IMAGE : value.replace(LOCAL_ORIGIN, '$1:<port>');
+    if (typeof value === 'number' || typeof value === 'boolean' || value === null)
+        return value;
+    if (Array.isArray(value))
+        return value.map((item) => (item === undefined ? null : canonicalRequest(item)));
+    if (isPlainObject(value)) {
+        if (value.type === 'image_url')
+            return IMAGE;
+        const keys = Object.keys(value)
+            .filter((key) => value[key] !== undefined)
+            .sort();
+        return Object.fromEntries(keys.map((key) => [key, canonicalRequest(value[key])]));
+    }
+    const json = value?.toJSON?.();
+    return json === undefined ? null : canonicalRequest(json);
+};
+const keyOf = (model, canonical) => createHash('sha256').update(JSON.stringify({ model, request: canonical })).digest('hex');
+/**
+ * The replay key: the sha256 of the model id and the canonical request. The same page asked of another model never
+ * matches.
+ *
+ * @example
+ * requestKey('jev-1.14.0', request) === requestKey('jev-1.14.0', sameStepOnAnotherPort); // true
+ */
+export const requestKey = (model, request) => keyOf(model, canonicalRequest(request));
+let recordings = 0;
+/**
+ * Wraps the live Ask: each answer is appended to this process's own JSONL file in `dir` (a header line with the route
+ * first, then one `{ key, model, request, answer }` line per request), in the order the answers came. A failed request
+ * is not recorded. The request is stored as sent, already scrubbed of secret values, with its images left out.
+ *
+ * @example
+ * const ask = recordingAsk(liveAsk, { profile: 'jev', model: 'jev-1.14.0', route: 'direct' }, 'verify/out/replay/mine/gotchas');
+ */
+export const recordingAsk = (ask, route, dir = replayDir()) => {
+    const file = join(dir, `${new Date().toISOString().replace(/[:.]/g, '-')}-${process.pid}-${++recordings}.jsonl`);
+    let started = false;
+    const write = (line) => appendFileSync(file, `${JSON.stringify(line)}\n`);
+    return async (request, options) => {
+        const answer = await ask(request, options);
+        if (!started) {
+            mkdirSync(dir, { recursive: true });
+            write({ header: route });
+            started = true;
+        }
+        const canonical = canonicalRequest(request);
+        write({ key: keyOf(route.model, canonical), model: route.model, request: canonical, answer });
+        return answer;
+    };
+};
+const recordingFiles = (dir) => existsSync(dir)
+    ? readdirSync(dir)
+        .filter((file) => file.endsWith('.jsonl'))
+        .sort()
+        .map((file) => join(dir, file))
+    : [];
+const linesOf = (file) => readFileSync(file, 'utf8')
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => JSON.parse(line));
+const nothingRecorded = (dir) => new Error(`No recorded Jev answers in ${dir}; record them first with JEVWAY_REPLAY=record.`);
+/**
+ * The route the recordings in `dir` were made on (the first file's header), so a replay resolves the same profile.
+ *
+ * @example
+ * recordedRoute('verify/out/replay/mine/gotchas'); // { profile: 'jev', model: 'jev-1.14.0', route: 'direct' }
+ */
+export const recordedRoute = (dir = replayDir()) => {
+    for (const file of recordingFiles(dir)) {
+        const header = linesOf(file).find((line) => 'header' in line);
+        if (header)
+            return header.header;
+    }
+    throw nothingRecorded(dir);
+};
+/**
+ * An Ask that answers only from the recordings in `dir` and never reaches the network: answers come back at once.
+ * A request recorded more than once gets its answers in the order they were recorded, then the last one again. A
+ * request never recorded throws `ReplayMiss`, naming its goal, step and kind with a diff against the nearest recorded
+ * request, and writes one `replay miss:` line to stderr, because a run may end a failed request as a hand-back.
+ *
+ * @example
+ * const ask = replayingAsk('jev-1.14.0', 'verify/out/replay/mine/gotchas');
+ */
+export const replayingAsk = (model, dir = replayDir()) => {
+    const entries = recordingFiles(dir)
+        .flatMap(linesOf)
+        .filter((line) => 'key' in line);
+    if (!entries.length)
+        throw nothingRecorded(dir);
+    const answers = new Map();
+    for (const entry of entries)
+        answers.set(entry.key, [...(answers.get(entry.key) ?? []), entry.answer]);
+    const served = new Map();
+    const ofModel = entries.filter((entry) => entry.model === model);
+    return async (request) => {
+        const canonical = canonicalRequest(request);
+        const key = keyOf(model, canonical);
+        const recorded = answers.get(key);
+        if (recorded) {
+            const turn = served.get(key) ?? 0;
+            served.set(key, turn + 1);
+            return structuredClone(recorded[Math.min(turn, recorded.length - 1)]);
+        }
+        const { headline, brief, paths, pageDiff } = describeMiss(canonical, model, ofModel);
+        process.stderr.write(`replay miss: ${headline} · ${brief}\n${[...paths, ...pageDiff].map((line) => `    ${line}\n`).join('')}`);
+        throw new ReplayMiss(`Replay miss: ${headline}\n${paths.join('\n')}`, pageDiff);
+    };
+};
+// What a request is about, for a person reading a miss: its goal, the step it was asked at, and its kind.
+const describeRequest = (canonical) => {
+    const request = canonical;
+    const questions = request.questions ?? {};
+    const state = (Array.isArray(request.state) ? request.state.find((part) => isPlainObject(part) && 'page' in part) : request.state);
+    const instructions = Object.values(questions).map((question) => question?.instructions);
+    const goal = instructions.find((text) => isPlainObject(text) && typeof text.goal === 'string')?.goal ??
+        instructions.find((text) => typeof text === 'string' && text.startsWith('Goal: '))?.split('\n')[0].slice('Goal: '.length) ??
+        instructions.find((text) => isPlainObject(text) && typeof text.completion_condition === 'string')?.completion_condition;
+    const ids = Object.keys(questions);
+    const kind = 'operation' in questions ? 'decision' : 'evidence' in questions ? 'evidence' : `follow-up (${ids.slice(0, 4).join(', ')})`;
+    const step = Array.isArray(state?.recent_actions) ? state.recent_actions.length + 1 : undefined;
+    return { goal: goal, kind, step, pageText: typeof state?.page?.text === 'string' ? state.page.text : undefined };
+};
+// Every leaf of a canonical request by its dotted path.
+const leaves = (value, path = '', into = new Map()) => {
+    if (value !== null && typeof value === 'object') {
+        for (const [key, inner] of Object.entries(value))
+            leaves(inner, path ? `${path}.${key}` : key, into);
+    }
+    else
+        into.set(path, JSON.stringify(value));
+    return into;
+};
+const changedPaths = (a, b) => [...new Set([...a.keys(), ...b.keys()])].filter((path) => a.get(path) !== b.get(path));
+const cut = (text, length) => (text.length > length ? `${text.slice(0, length - 1)}…` : text);
+const MAX_PATHS = 8;
+const MAX_LINES = 6;
+// The nearest recorded request with the same goal (any, when none shares it), and what differs from it.
+const describeMiss = (canonical, model, recorded) => {
+    const asked = describeRequest(canonical);
+    const headline = [`"${cut(asked.goal ?? 'no goal', 80)}"`, ...(asked.step ? [`step ${asked.step}`] : []), asked.kind].join(' · ');
+    if (!recorded.length)
+        return { headline, brief: `nothing was recorded for ${model}`, paths: [`nothing was recorded for ${model}`], pageDiff: [] };
+    const sameGoal = recorded.filter((entry) => describeRequest(entry.request).goal === asked.goal);
+    const askedLeaves = leaves(canonical);
+    const [nearest] = (sameGoal.length ? sameGoal : recorded)
+        .map((entry) => ({ entry, changed: changedPaths(leaves(entry.request), askedLeaves) }))
+        .sort((a, b) => a.changed.length - b.changed.length);
+    const paths = [
+        `nearest recorded request${sameGoal.length ? ' with the same goal' : ''} differs at ${nearest.changed.length} paths:`,
+        ...nearest.changed.slice(0, MAX_PATHS).map((path) => `  ${path}`),
+    ];
+    const pageDiff = [];
+    const before = describeRequest(nearest.entry.request).pageText;
+    if (before !== undefined && asked.pageText !== undefined && before !== asked.pageText) {
+        const [was, now] = [before.split('\n'), asked.pageText.split('\n')];
+        const onlyIn = (lines, other) => lines.filter((line) => !other.includes(line)).slice(0, MAX_LINES).map((line) => `  ${cut(line, 120)}`);
+        pageDiff.push('page text only recorded:', ...onlyIn(was, now), 'page text only now:', ...onlyIn(now, was));
+    }
+    return { headline, brief: `differs at ${nearest.changed.slice(0, 3).join(', ')}`, paths, pageDiff };
+};

package/dist/jev/secrets.js

@@ -0,0 +1,60 @@
+/** What stands in for a secret value wherever it would otherwise show: Jev's request, the caller's summary, traces, recordings. */
+export const secretMarker = (key) => `[secret:${key}]`;
+const secretEntries = (values, secretKeys) => Object.entries(values).filter(([key, value]) => secretKeys.includes(key) && value.trim() !== '');
+/**
+ * The supplied values as Jev may see them: each key the caller marked secret carries its marker, so Jev can still pick
+ * the field it belongs in by its name, and never reads the value. With nothing marked, the same object comes back.
+ *
+ * @example
+ * shownValues({ token: 'ghp_x1', repo: 'ada/app' }, ['token']) // { token: '[secret:token]', repo: 'ada/app' }
+ */
+export const shownValues = (values, secretKeys = []) => {
+    const secrets = secretEntries(values, secretKeys);
+    if (!secrets.length)
+        return values;
+    return { ...values, ...Object.fromEntries(secrets.map(([key]) => [key, secretMarker(key)])) };
+};
+// How a page may echo a value back: as typed, trimmed, or URL- and form-encoded in an address.
+const formsOf = (value) => {
+    const encoded = encodeURIComponent(value.trim());
+    return [value, value.trim(), encodeURIComponent(value), encoded, encoded.replace(/%20/g, '+')];
+};
+const escapeRegExp = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+const isPlainObject = (value) => typeof value === 'object' && value !== null && Object.getPrototypeOf(value) === Object.prototype;
+// An image part of a request is pixels: its base64 can contain a short secret by chance, and a replacement would corrupt it.
+const isImagePart = (value) => value.type === 'image_url';
+/**
+ * A function that replaces every secret value in a string, or in every string of a plain object or array (keys,
+ * numbers, class instances such as an image, and `image_url` parts are left alone), with its key's marker. The business rule: a value the
+ * caller marked secret never reaches the model or any output, including when the page echoes it (upper-cased, in an
+ * address). Matching is case-insensitive and takes the longest form first, in one pass, so a marker is never scrubbed
+ * again. Every non-empty value counts, even a short one: a garbled text is the safe failure, a leaked secret is not.
+ * With nothing marked it is the identity.
+ *
+ * @example
+ * const scrub = scrubberFor({ token: 'ghp_x1' }, ['token']);
+ * scrub({ page_text: 'Token GHP_X1 created' }) // { page_text: 'Token [secret:token] created' }
+ */
+export const scrubberFor = (values, secretKeys = []) => {
+    const keyByForm = new Map();
+    for (const [key, value] of secretEntries(values, secretKeys)) {
+        for (const form of formsOf(value))
+            if (form && !keyByForm.has(form.toLowerCase()))
+                keyByForm.set(form.toLowerCase(), key);
+    }
+    if (!keyByForm.size)
+        return (value) => value;
+    const forms = [...keyByForm.keys()].sort((a, b) => b.length - a.length);
+    const pattern = new RegExp(forms.map(escapeRegExp).join('|'), 'gi');
+    const scrubText = (text) => text.replace(pattern, (match) => secretMarker(keyByForm.get(match.toLowerCase())));
+    const scrub = (value) => {
+        if (typeof value === 'string')
+            return scrubText(value);
+        if (Array.isArray(value))
+            return value.map(scrub);
+        if (isPlainObject(value) && !isImagePart(value))
+            return Object.fromEntries(Object.entries(value).map(([key, inner]) => [key, scrub(inner)]));
+        return value;
+    };
+    return (value) => scrub(value);
+};

package/dist/journeys/diagnostics.js

@@ -0,0 +1,85 @@
+const MAX_ENTRIES = 20;
+const clip = (text, length = 300) => (text.length > length ? `${text.slice(0, length)}…` : text);
+/**
+ * Collects console errors, uncaught exceptions and failed requests for a tab, and tracks in-flight
+ * requests so a screenshot waits for the screen to finish loading.
+ *
+ * @example
+ * const diagnostics = await watchDiagnostics(tab);
+ * await tab.act(action, page);
+ * await diagnostics.waitForIdle();
+ * const { consoleErrors, failedRequests } = diagnostics.drain(); // errors since the last drain
+ */
+export const watchDiagnostics = async (tab) => {
+    let current = { consoleErrors: [], failedRequests: [] };
+    const urls = new Map();
+    const inflight = new Set();
+    let lastActivity = Date.now();
+    const addError = (text) => {
+        if (current.consoleErrors.length < MAX_ENTRIES)
+            current.consoleErrors.push(clip(text));
+    };
+    const addFailure = (failure) => {
+        // Ad blockers and cancelled navigations are the viewer's environment, not the site's bugs.
+        const environmental = failure.error === 'net::ERR_BLOCKED_BY_CLIENT' || failure.error === 'net::ERR_ABORTED';
+        if (environmental || failure.url.startsWith('data:') || current.failedRequests.length >= MAX_ENTRIES)
+            return;
+        if (current.failedRequests.some((existing) => existing.url === clip(failure.url, 200)))
+            return;
+        current.failedRequests.push({ ...failure, url: clip(failure.url, 200) });
+    };
+    const unsubscribe = tab.onEvent((method, params) => {
+        switch (method) {
+            case 'Runtime.consoleAPICalled':
+                if (params.type === 'error')
+                    addError(params.args.map((arg) => arg.value ?? arg.description).join(' '));
+                break;
+            case 'Runtime.exceptionThrown':
+                addError(params.exceptionDetails.exception?.description ?? params.exceptionDetails.text);
+                break;
+            case 'Network.requestWillBeSent':
+                urls.set(params.requestId, params.request.url);
+                inflight.add(params.requestId);
+                lastActivity = Date.now();
+                break;
+            case 'Network.responseReceived':
+                if (params.response.status >= 400)
+                    addFailure({ url: params.response.url, status: params.response.status });
+                break;
+            case 'Network.loadingFinished':
+            case 'Network.loadingFailed':
+                inflight.delete(params.requestId);
+                lastActivity = Date.now();
+                if (method === 'Network.loadingFailed' && !params.canceled) {
+                    addFailure({ url: urls.get(params.requestId) ?? '', error: params.errorText });
+                }
+                break;
+        }
+    });
+    await Promise.all([tab.send('Runtime.enable'), tab.send('Network.enable')]);
+    return {
+        /** Returns everything collected since the previous drain. */
+        drain() {
+            const collected = current;
+            current = { consoleErrors: [], failedRequests: [] };
+            return collected;
+        },
+        /** Waits until the network is quiet, then for visible images to decode. Never waits longer than maxMs. */
+        async waitForIdle({ quietMs = 400, maxMs = 4000 } = {}) {
+            const deadline = Date.now() + maxMs;
+            // Long-polling and analytics beacons can keep a request open forever; quiet time still counts past 2 in flight.
+            while (Date.now() < deadline && (inflight.size > 2 || Date.now() - lastActivity < quietMs)) {
+                await new Promise((resolve) => setTimeout(resolve, 50));
+            }
+            await tab
+                .evaluate(`Promise.race([
+            Promise.all([...document.images]
+              .filter(i => !i.complete && i.getBoundingClientRect().top < innerHeight)
+              .map(i => new Promise(r => { i.onload = i.onerror = r; }))),
+            new Promise(r => setTimeout(r, ${Math.max(0, Math.min(1500, deadline - Date.now()))}))
+          ]).then(() => true)`)
+                .catch(() => undefined);
+        },
+        stop: unsubscribe,
+    };
+};

package/dist/journeys/journey-doc.js

@@ -0,0 +1,50 @@
+export const hotspotOf = (action, page) => {
+    if (!action.rect || !page.w || !page.h)
+        return null;
+    const clamp = (value) => Math.min(1, Math.max(0, value));
+    return {
+        x: clamp(action.rect.x / page.w),
+        y: clamp(action.rect.y / page.h),
+        width: clamp(action.rect.w / page.w),
+        height: clamp(action.rect.h / page.h),
+    };
+};
+/**
+ * Groups steps into named flows in first-seen order. A flow name like "Homes/Searching homes" nests under
+ * "Homes". Consecutive steps chain screens; a gap (the flow resumes from another screen) starts a new segment
+ * inside the same flow.
+ *
+ * @example
+ * buildFlows([{ from: 's01', to: 's02', flow: 'Search', action: { operation: 'CLICK', label: 'Search' }, … }])
+ * // [{ name: 'Search', parent: null, screens: [{ screen: 's01', action: 'Tap "Search"', hotspot }, { screen: 's02' }] }]
+ */
+export const buildFlows = (steps) => {
+    const flows = new Map();
+    for (const step of steps) {
+        const flow = flows.get(step.flow) ?? { name: step.flow, parent: parentOf(step.flow), screens: [] };
+        flows.set(step.flow, flow);
+        if (flow.screens.at(-1)?.screen !== step.from)
+            flow.screens.push({ screen: step.from });
+        const current = flow.screens.at(-1);
+        // Several interactions on one screen (type, pick a suggestion, then submit) read as one card.
+        current.action = current.action ? `${current.action}, then ${describeAction(step)}` : describeAction(step);
+        if (step.hotspot)
+            current.hotspot = step.hotspot;
+        if (step.to !== step.from)
+            flow.screens.push({ screen: step.to });
+    }
+    // Parents that only exist as a prefix still appear in the tree.
+    for (const flow of [...flows.values()]) {
+        let parent = flow.parent;
+        while (parent && !flows.has(parent)) {
+            flows.set(parent, { name: parent, parent: parentOf(parent), screens: [] });
+            parent = parentOf(parent);
+        }
+    }
+    return [...flows.values()];
+};
+const parentOf = (name) => (name.includes('/') ? name.slice(0, name.lastIndexOf('/')) : null);
+const describeAction = (step) => {
+    const verb = { CLICK: 'Tap', TYPE_TEXT: 'Type into', SELECT: 'Choose' }[step.action.operation] ?? step.action.operation;
+    return `${verb} "${step.action.label}"${step.action.value ? ` → "${step.action.value}"` : ''}`;
+};