@reticlehq/server 3.9.0 → 3.10.0

package/dist/features/compare/behaviour-diff.d.ts

@@ -0,0 +1,97 @@
+/**
+ * The verdict of `reticle compare`: what the candidate shows that the reference does not. Pure.
+ *
+ * The reference is read two times. A line that differs between those two runs is noise (a clock, a
+ * random id) and is ignored in that sequence. A sequence the reference did on only one of its runs
+ * is not compared. A step the candidate cannot do where the reference could is a difference.
+ * Differences with the same lines are one difference, with one id and the number of sequences it
+ * was seen in.
+ *
+ * With a base build (the app before the change), each difference also gets a tag:
+ * - `regression`: base and reference show the same page, and the candidate changed it.
+ * - `target-missed`: the reference changed the page, and the candidate still shows the base page.
+ * - `changed`: the candidate differs from both, or the base did not show this sequence.
+ * The score is targets matched / targets x (1 - kept sequences changed / kept). A target is a
+ * sequence where the reference changed the base page; a kept sequence is one where it did not.
+ */
+import { DifferenceKind } from './compare-diff.js';
+import type { BuildReadings } from './build-readings.js';
+import type { PlanDropped } from './action-plan.js';
+export declare const CompareVerdict: {
+    readonly SAME: "same";
+    readonly DIFFERENT: "different";
+    /** The run proves nothing: the driver lost its session, or nothing could be compared. */
+    readonly INVALID: "invalid";
+};
+export type CompareVerdict = (typeof CompareVerdict)[keyof typeof CompareVerdict];
+export declare const DifferenceTag: {
+    readonly REGRESSION: "regression";
+    readonly TARGET_MISSED: "target-missed";
+    readonly CHANGED: "changed";
+};
+export type DifferenceTag = (typeof DifferenceTag)[keyof typeof DifferenceTag];
+export interface BehaviourDifference {
+    /** A hash of the difference's lines: the same lines give the same id on every run. */
+    id: string;
+    kind: DifferenceKind;
+    /** The steps that first reached it. */
+    actions: string[];
+    /** Lines only the reference shows. */
+    removed: string[];
+    /** Lines only the candidate shows. */
+    added: string[];
+    /** For `unreached`: the step the candidate could not do. */
+    failedAt?: string;
+    /** In how many sequences this same difference was seen. */
+    seen: number;
+    /** With a base build only. */
+    tag?: DifferenceTag;
+}
+export interface BehaviourScore {
+    /** Null when the plan reached no target: the score cannot see the change. */
+    score: number | null;
+    targets: number;
+    matched: number;
+    kept: number;
+    changed: number;
+}
+export interface BehaviourComparison {
+    verdict: CompareVerdict;
+    /** Planned sequences. */
+    sequences: number;
+    /** Sequences both builds showed, and were compared. */
+    compared: number;
+    /** Sequences the reference did on only one of its two runs, so they were not compared. */
+    unstable: number;
+    /** Sequences the reference could not do on its first run, so they were not compared. */
+    referenceFailed: number;
+    /** Lines that differed but were ignored, because they also differed between the reference runs. */
+    ignored: number;
+    differences: BehaviourDifference[];
+    score?: BehaviourScore;
+    /** What the plan left out because of a cap. Absent when no plan was made. */
+    planDropped?: PlanDropped;
+    /** Why the run is invalid. */
+    reason?: string;
+    /**
+     * Sequences where a candidate step failed once and was done on a retry from a fresh page. They do not change the
+     * verdict, and they are never hidden: a fault that comes and goes is reported here. Absent when there were none.
+     */
+    intermittent?: {
+        actions: string[];
+        step: string;
+    }[];
+}
+export declare const MSG_NOTHING_COMPARED: string;
+/** A comparison that proves nothing. */
+export declare function invalidComparison(reason: string, sequences: number): BehaviourComparison;
+interface Input {
+    /** The sequence keys, in plan order. */
+    order: readonly string[];
+    reference: BuildReadings;
+    again: BuildReadings;
+    candidate: BuildReadings;
+    base?: BuildReadings;
+}
+export declare function compareBehaviour(input: Input): BehaviourComparison;
+export {};

package/dist/features/compare/behaviour-diff.js

@@ -0,0 +1,202 @@
+/**
+ * The verdict of `reticle compare`: what the candidate shows that the reference does not. Pure.
+ *
+ * The reference is read two times. A line that differs between those two runs is noise (a clock, a
+ * random id) and is ignored in that sequence. A sequence the reference did on only one of its runs
+ * is not compared. A step the candidate cannot do where the reference could is a difference.
+ * Differences with the same lines are one difference, with one id and the number of sequences it
+ * was seen in.
+ *
+ * With a base build (the app before the change), each difference also gets a tag:
+ * - `regression`: base and reference show the same page, and the candidate changed it.
+ * - `target-missed`: the reference changed the page, and the candidate still shows the base page.
+ * - `changed`: the candidate differs from both, or the base did not show this sequence.
+ * The score is targets matched / targets x (1 - kept sequences changed / kept). A target is a
+ * sequence where the reference changed the base page; a kept sequence is one where it did not.
+ */
+import { DifferenceKind, differenceId, lineDiff, maskedDiff } from './compare-diff.js';
+import { isLoadedRead } from './outside/effect-lines.js';
+export const CompareVerdict = {
+    SAME: 'same',
+    DIFFERENT: 'different',
+    /** The run proves nothing: the driver lost its session, or nothing could be compared. */
+    INVALID: 'invalid',
+};
+export const DifferenceTag = {
+    REGRESSION: 'regression',
+    TARGET_MISSED: 'target-missed',
+    CHANGED: 'changed',
+};
+export const MSG_NOTHING_COMPARED = 'Nothing was compared, so nothing was proved. The reference did no sequence two times the same way. ' +
+    'Make sure the reference url opens, and that its page does not change on each load.';
+/** A comparison that proves nothing. */
+export function invalidComparison(reason, sequences) {
+    return {
+        verdict: CompareVerdict.INVALID,
+        sequences,
+        compared: 0,
+        unstable: 0,
+        referenceFailed: 0,
+        ignored: 0,
+        differences: [],
+        reason,
+    };
+}
+const DIGITS = /\d+/g;
+const HAS_DIGIT = /\d/;
+const DIGIT_MARK = '#';
+/** A line with its numbers folded: `ticker: 76` and `ticker: 77` have one shape. */
+const shapeOf = (line) => line.replace(DIGITS, DIGIT_MARK);
+const runNoise = (ours, repeat) => {
+    if (ours?.lines === undefined || repeat?.lines === undefined) {
+        return [];
+    }
+    const d = lineDiff(ours.lines, repeat.lines);
+    return [...d.removed, ...d.added];
+};
+/**
+ * What is noise in every sequence: a read that loaded (`isLoadedRead`), and the shapes of the noise lines that
+ * hold a number, over every sequence. A live counter read the
+ * same on both reference runs in one sequence by chance, and was then a difference there. When the
+ * reference runs showed it changing in any sequence, it is noise in all of them.
+ */
+function counterShapes(input) {
+    const shapes = new Set();
+    for (const key of input.order) {
+        for (const line of runNoise(input.reference.get(key), input.again.get(key))) {
+            if (HAS_DIGIT.test(line)) {
+                shapes.add(shapeOf(line));
+            }
+        }
+    }
+    return (line) => shapes.has(shapeOf(line)) || isLoadedRead(line);
+}
+/** The noise of one sequence: what its reference runs disagree on, and every counter shape. */
+const noiseOf = (ours, repeat, counters) => {
+    const own = new Set(runNoise(ours, repeat));
+    return (line) => own.has(line) || counters(line);
+};
+/** The visible page without its noise lines, as one string, or undefined when it was not shown. */
+const masked = (reading, isNoise) => reading?.lines?.filter((line) => !isNoise(line)).join('\n');
+function tagOf(input, key, noise) {
+    const b = masked(input.base?.get(key), noise);
+    if (b === undefined) {
+        return DifferenceTag.CHANGED;
+    }
+    if (b === masked(input.reference.get(key), noise)) {
+        return DifferenceTag.REGRESSION;
+    }
+    return b === masked(input.candidate.get(key), noise)
+        ? DifferenceTag.TARGET_MISSED
+        : DifferenceTag.CHANGED;
+}
+function scoreOf(input, base, counters) {
+    let targets = 0;
+    let matched = 0;
+    let kept = 0;
+    let changed = 0;
+    for (const key of input.order) {
+        const ours = input.reference.get(key);
+        const repeat = input.again.get(key);
+        if (ours?.lines === undefined || repeat?.lines === undefined) {
+            continue;
+        }
+        const noise = noiseOf(ours, repeat, counters);
+        const r = masked(ours, noise);
+        const b = masked(base.get(key), noise);
+        if (b === undefined) {
+            continue;
+        }
+        const got = masked(input.candidate.get(key), noise);
+        if (b === r) {
+            kept += 1;
+            changed += got === r ? 0 : 1;
+        }
+        else {
+            targets += 1;
+            matched += got === r ? 1 : 0;
+        }
+    }
+    const progress = 0 === targets ? null : matched / targets;
+    const score = null === progress ? null : progress * (0 === kept ? 1 : 1 - changed / kept);
+    return { score, targets, matched, kept, changed };
+}
+export function compareBehaviour(input) {
+    const counters = counterShapes(input);
+    const found = new Map();
+    let compared = 0;
+    let unstable = 0;
+    let referenceFailed = 0;
+    let ignored = 0;
+    const intermittent = [];
+    const note = (key, difference) => {
+        const id = differenceId(DifferenceKind.UNREACHED === difference.kind
+            ? { unreached: difference.failedAt }
+            : { removed: difference.removed, added: difference.added });
+        const known = found.get(id);
+        if (known !== undefined) {
+            known.seen += 1;
+            return;
+        }
+        const tag = input.base === undefined
+            ? undefined
+            : tagOf(input, key, noiseOf(input.reference.get(key), input.again.get(key), counters));
+        found.set(id, { id, seen: 1, ...difference, ...(tag === undefined ? {} : { tag }) });
+    };
+    for (const key of input.order) {
+        const ours = input.reference.get(key);
+        if (ours?.lines === undefined) {
+            referenceFailed += 1;
+            continue;
+        }
+        const repeat = input.again.get(key)?.lines;
+        if (repeat === undefined) {
+            unstable += 1;
+            continue;
+        }
+        const got = input.candidate.get(key);
+        if (got?.retriedAt !== undefined) {
+            intermittent.push({ actions: got.actions, step: got.retriedAt });
+        }
+        if (got?.lines === undefined) {
+            const failedAt = got?.failedAt ?? ours.actions.at(-1) ?? '';
+            note(key, {
+                kind: DifferenceKind.UNREACHED,
+                actions: ours.actions,
+                removed: [],
+                added: [],
+                failedAt,
+            });
+            continue;
+        }
+        compared += 1;
+        const diff = maskedDiff(ours.lines, repeat, got.lines, counters);
+        ignored += diff.ignored;
+        if (diff.removed.length > 0 || diff.added.length > 0) {
+            note(key, {
+                kind: DifferenceKind.CHANGED,
+                actions: ours.actions,
+                removed: diff.removed,
+                added: diff.added,
+            });
+        }
+    }
+    const differences = [...found.values()];
+    const counts = {
+        sequences: input.order.length,
+        compared,
+        unstable,
+        referenceFailed,
+        ignored,
+    };
+    if (0 === compared && 0 === differences.length) {
+        return { ...invalidComparison(MSG_NOTHING_COMPARED, input.order.length), ...counts };
+    }
+    return {
+        verdict: 0 === differences.length ? CompareVerdict.SAME : CompareVerdict.DIFFERENT,
+        ...counts,
+        differences,
+        ...(input.base === undefined ? {} : { score: scoreOf(input, input.base, counters) }),
+        ...(0 === intermittent.length ? {} : { intermittent }),
+    };
+}

package/dist/features/compare/behaviour-report.d.ts

@@ -0,0 +1,40 @@
+/**
+ * What `reticle compare` prints and exits with.
+ *
+ * The text shows the verdict first, then the counts, the score when a base was given, and the top
+ * differences. Each difference shows the steps that reproduce it and its `-` and `+` lines. The text
+ * shows the first `TOP_DIFFERENCES` differences and the first `MAX_DIFFERENCE_LINES` lines of each,
+ * and says how many it left out. `--json` gives everything.
+ */
+import { Verified } from '@reticlehq/core';
+import { type BehaviourComparison } from './behaviour-diff.js';
+export declare const CompareExit: {
+    readonly SAME: 0;
+    readonly DIFFERENT: 1;
+    readonly INVALID: 2;
+};
+export declare const COMPARE_SCHEMA_VERSION = 1;
+export declare const TOP_DIFFERENCES = 5;
+export declare const MAX_DIFFERENCE_LINES = 12;
+export interface CompareUrls {
+    reference: string;
+    candidate: string;
+    base?: string;
+}
+/** Seconds spent on each part of the run. */
+export interface CompareTimings {
+    plan: number;
+    reference: number;
+    candidate: number;
+    base?: number;
+    again: number;
+}
+export declare const compareExitCode: (result: BehaviourComparison) => number;
+export declare function compareReportLines(urls: CompareUrls, result: BehaviourComparison, timings?: CompareTimings): string[];
+/** The `--json` object: every difference and every count the text gives. */
+export declare function compareReportJson(urls: CompareUrls, result: BehaviourComparison, timings?: CompareTimings): CompareUrls & BehaviourComparison & {
+    schemaVersion: number;
+    verified: Verified;
+    exitCode: number;
+    seconds?: CompareTimings;
+};

package/dist/features/compare/behaviour-report.js

@@ -0,0 +1,128 @@
+/**
+ * What `reticle compare` prints and exits with.
+ *
+ * The text shows the verdict first, then the counts, the score when a base was given, and the top
+ * differences. Each difference shows the steps that reproduce it and its `-` and `+` lines. The text
+ * shows the first `TOP_DIFFERENCES` differences and the first `MAX_DIFFERENCE_LINES` lines of each,
+ * and says how many it left out. `--json` gives everything.
+ */
+import { Verified } from '@reticlehq/core';
+import { NOTHING_DROPPED } from './action-plan.js';
+import { DifferenceKind } from './compare-diff.js';
+import { CompareVerdict, } from './behaviour-diff.js';
+export const CompareExit = { SAME: 0, DIFFERENT: 1, INVALID: 2 };
+export const COMPARE_SCHEMA_VERSION = 1;
+export const TOP_DIFFERENCES = 5;
+export const MAX_DIFFERENCE_LINES = 12;
+const INDENT = '  ';
+const SCORE_DIGITS = 2;
+const VERDICT_WORD = {
+    [CompareVerdict.SAME]: 'SAME',
+    [CompareVerdict.DIFFERENT]: 'DIFFERENT',
+    [CompareVerdict.INVALID]: 'INVALID',
+};
+const EXIT = {
+    [CompareVerdict.SAME]: CompareExit.SAME,
+    [CompareVerdict.DIFFERENT]: CompareExit.DIFFERENT,
+    [CompareVerdict.INVALID]: CompareExit.INVALID,
+};
+export const compareExitCode = (result) => EXIT[result.verdict];
+/** The verdict in Reticle's own word: a run that proves nothing is `unknown`, never `no`. */
+const VERIFIED = {
+    [CompareVerdict.SAME]: Verified.YES,
+    [CompareVerdict.DIFFERENT]: Verified.NO,
+    [CompareVerdict.INVALID]: Verified.UNKNOWN,
+};
+const plural = (n, one, many) => `${String(n)} ${1 === n ? one : many}`;
+function headline(urls, result) {
+    const word = VERDICT_WORD[result.verdict];
+    if (CompareVerdict.INVALID === result.verdict) {
+        return `${word}: ${result.reason ?? ''}`;
+    }
+    if (CompareVerdict.SAME === result.verdict) {
+        return `${word}: ${urls.candidate} shows what ${urls.reference} shows after every sequence.`;
+    }
+    return `${word}: ${urls.candidate} does not show what ${urls.reference} shows. ${plural(result.differences.length, 'difference', 'differences')}.`;
+}
+function timingLine(t) {
+    const parts = [
+        `plan ${String(t.plan)} s`,
+        `reference ${String(t.reference)} s`,
+        `candidate ${String(t.candidate)} s`,
+        ...(t.base === undefined ? [] : [`base ${String(t.base)} s`]),
+        `reference again ${String(t.again)} s`,
+    ];
+    return `Time: ${parts.join(', ')}.`;
+}
+function scoreLine(result) {
+    const s = result.score;
+    if (s === undefined) {
+        return undefined;
+    }
+    const tail = `${String(s.matched)} of ${String(s.targets)} targets matched, ${String(s.changed)} of ${String(s.kept)} kept sequences changed.`;
+    if (null === s.score) {
+        return `score: none. The plan reached no page that the reference changed from the base. ${tail}`;
+    }
+    return `score ${s.score.toFixed(SCORE_DIGITS)}: ${tail}`;
+}
+function differenceLines(index, d) {
+    const tag = d.tag === undefined ? '' : `[${d.tag}] `;
+    const where = plural(d.seen, 'sequence', 'sequences');
+    const out = [`${String(index + 1)}. ${tag}difference ${d.id} (seen in ${where})`];
+    out.push(...d.actions.map((a, i) => `${INDENT}${INDENT}${String(i + 1)}. ${a}`));
+    if (DifferenceKind.UNREACHED === d.kind) {
+        out.push(`${INDENT}The candidate could not do this step. The reference could: ${d.failedAt ?? ''}`);
+    }
+    const marked = [...d.removed.map((l) => `- ${l}`), ...d.added.map((l) => `+ ${l}`)];
+    out.push(...marked.slice(0, MAX_DIFFERENCE_LINES).map((l) => `${INDENT}${l}`));
+    if (marked.length > MAX_DIFFERENCE_LINES) {
+        out.push(`${INDENT}... ${String(marked.length - MAX_DIFFERENCE_LINES)} more lines (--json prints all)`);
+    }
+    return out;
+}
+export function compareReportLines(urls, result, timings) {
+    const lines = [headline(urls, result)];
+    if (result.sequences > 0) {
+        lines.push(`${plural(result.sequences, 'sequence', 'sequences')}, ${String(result.compared)} compared, ` +
+            `${String(result.unstable)} not compared (the reference did them on one of its two runs only), ` +
+            `${String(result.referenceFailed)} the reference could not do, ` +
+            `${String(result.ignored)} lines ignored as noise.`);
+    }
+    const dropped = Object.values(result.planDropped ?? NOTHING_DROPPED);
+    const capped = dropped.reduce((sum, n) => sum + n, 0);
+    if (capped > 0) {
+        lines.push(`The plan left out ${plural(capped, 'item', 'items')} at its caps. --json "planDropped" gives the count for each cap.`);
+    }
+    const score = scoreLine(result);
+    if (score !== undefined) {
+        lines.push(score);
+    }
+    if (timings !== undefined) {
+        lines.push(timingLine(timings));
+    }
+    // A retry never hides a fault that comes and goes: each such step is listed, whatever the verdict.
+    const flaky = result.intermittent ?? [];
+    if (flaky.length > 0) {
+        lines.push('', `${plural(flaky.length, 'step', 'steps')} failed once on the candidate and was done on a retry (check for a fault that comes and goes):`, ...flaky.map((f) => `${INDENT}${f.step}  after: ${f.actions.join(' > ')}`));
+    }
+    const shown = result.differences.slice(0, TOP_DIFFERENCES);
+    for (const [i, d] of shown.entries()) {
+        lines.push('', ...differenceLines(i, d));
+    }
+    const left = result.differences.length - shown.length;
+    if (left > 0) {
+        lines.push('', `${plural(left, 'more difference', 'more differences')} (--json prints all).`);
+    }
+    return lines;
+}
+/** The `--json` object: every difference and every count the text gives. */
+export function compareReportJson(urls, result, timings) {
+    return {
+        schemaVersion: COMPARE_SCHEMA_VERSION,
+        ...urls,
+        ...result,
+        verified: VERIFIED[result.verdict],
+        exitCode: compareExitCode(result),
+        ...(timings === undefined ? {} : { seconds: timings }),
+    };
+}

package/dist/features/compare/build-readings.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Drive one build through an action plan and read what it shows after each planned sequence.
+ *
+ * The page is read only after it settles: read, wait, read again, until two reads in a row match,
+ * up to `SETTLE_READS` reads. A page that never settles (a slideshow) gives its last read.
+ *
+ * A sequence that starts another planned sequence is not driven on its own: the longer one is
+ * driven, and the page is read after each of its steps. Each planned sequence is read once, in the
+ * first run that reaches it, and the order is the same on every build.
+ *
+ * Each reading also holds what the app did since the sequence's first action: its network calls and
+ * console errors and warnings (`effect-lines.ts`), read from the cursor that first action returned.
+ *
+ * A step that could not be done (the control is not there, or the action was refused) is recorded
+ * for that sequence and each longer one in the run. A driver that lost its browser session throws
+ * `DriverLostError`: that says nothing about the build, so the caller makes the whole run invalid.
+ */
+import { ActionType } from '@reticlehq/core';
+import { type Probe, type Sequence } from './action-plan.js';
+/** Visible text with no ref: the act tool finds the element by its text. */
+export interface TextAnchor {
+    text: string;
+}
+/** One request fault, in the shape `reticle_network_mock` takes. */
+export interface FaultRule {
+    urlContains: string;
+    method: string;
+    status?: number;
+    body?: string;
+    abort?: boolean;
+}
+/** A browser tab on one build. */
+export interface PageDriver {
+    /** Load the start page fresh. False when it did not load. */
+    open(): Promise<boolean>;
+    /**
+     * Open the start page with `query` (`?scenario=deferred`) on the same tab. False when it did not
+     * load. A driver without it cannot do a step that opens a url parameter value.
+     */
+    goto?(query: string): Promise<boolean>;
+    /**
+     * Make requests fail on this tab, until the next `open`, which clears them. False when the faults
+     * were not set. A driver without it cannot do a step that makes a request fail.
+     */
+    mock?(rules: readonly FaultRule[]): Promise<boolean>;
+    /** The whole page as an accessibility tree, with refs and test ids. */
+    read(): Promise<string>;
+    /**
+     * Act on a control by ref, or click a text probe by its text. `ok` is false when the action was
+     * refused or did not land. `since` is the event cursor from before the action, when the driver
+     * gives one.
+     */
+    act(at: string | TextAnchor, action: 'click' | 'fill' | 'select' | typeof ActionType.PRESS, value?: string): Promise<{
+        ok: boolean;
+        since?: number;
+    }>;
+    /** What the app did since the cursor, as comparable lines (`effect-lines.ts`). */
+    effects(since: number): Promise<string[]>;
+}
+/** The driver lost its browser session. The run says nothing about the build. */
+export declare class DriverLostError extends Error {
+    name: string;
+}
+/** What one build showed after one planned sequence. */
+export interface Reading {
+    /** The steps that reach it, as a person reads them, starting with the start page. */
+    actions: string[];
+    /** The visible lines. Absent when a step could not be done. */
+    lines?: string[];
+    /** The step that could not be done. */
+    failedAt?: string;
+    /** The step that failed once and was done on a retry from a fresh page. The comparison reports it. */
+    retriedAt?: string;
+}
+/** Readings by sequence key. */
+export type BuildReadings = Map<string, Reading>;
+export declare const SETTLE_READS = 4;
+export declare const SETTLE_GAP_MS = 400;
+/** The first action of every sequence. */
+export declare const START_STEP = "open the start page";
+type Sleep = (ms: number) => Promise<void>;
+/**
+ * What a person sees on the page, one line for each element, with the role of each control and
+ * structure element kept (`structureLines`) and sub-second times rounded.
+ */
+export declare function pageLines(tree: string): string[];
+/** Read the page once it stops changing. */
+export declare function settledRead(driver: PageDriver, sleep: Sleep): Promise<{
+    tree: string;
+    lines: string[];
+    settled: boolean;
+}>;
+/**
+ * Drive one sequence from a fresh load, for the plan, or undefined when a step could not be done.
+ *
+ * For a sequence that ends in a wait, the page is also read, settled, before the wait. The wait is
+ * `active` when the page did not settle before it, a network call was still pending before it, or
+ * the page changed during it.
+ */
+export declare function probeSequence(driver: PageDriver, seq: Sequence, sleep: Sleep): Promise<Probe | undefined>;
+/**
+ * Drive the plan on one build. With `readState`, the app's stored data is read after each sequence
+ * is read, and its lines (`outside/state-sources.ts`) follow the page and effect lines.
+ */
+export declare function readBuild(driver: PageDriver, plan: readonly Sequence[], sleep: Sleep, readState?: () => Promise<string[]>): Promise<BuildReadings>;
+export {};