@setsquare/review-sdk 0.1.0 → 0.2.0

package/dist/summary.js

@@ -1,5 +1,5 @@
 import { conformanceGrid, conformanceSummary, ruleSurfaceShare, stateCoverage, } from '@setsquare/schema/conformance';
-import { PLATFORM_URL, ruleHelpUrl } from './links.js';
+import { ruleHelpUrl } from './links.js';
 import { continuationLines, createStyle, figureBlock, figureLine, heading, } from './style.js';
 import { flakeNotes, insightBlocks } from './insights.js';
 /**
@@ -120,6 +120,7 @@ export function groupFindings(envelope) {
                 ruleId: finding.ruleId,
                 impact: finding.impact ?? 'unknown',
                 status: finding.status,
+                ...(finding.undecidable ? { undecidable: finding.undecidable } : {}),
                 nodes: finding.targets.nodeCount,
                 contexts: [label],
                 contextNodes: [{ label, nodes: finding.targets.nodeCount }],
@@ -230,6 +231,8 @@ export function evidenceQuality(envelope) {
     let quiet = 0;
     let teardown = 0;
     let ceiling = 0;
+    let ceilingWaitedMs = 0;
+    let ceilingMutations = 0;
     for (const scan of envelope.scans) {
         for (const pass of scan.passes ?? [])
             rules.add(pass.ruleId);
@@ -246,12 +249,17 @@ export function evidenceQuality(envelope) {
                 break;
             case 'ceiling':
                 ceiling += 1;
+                // The worst case rather than an average: one page pinned at the ceiling
+                // is the thing worth knowing about, and a mean over a healthy run would
+                // hide it.
+                ceilingWaitedMs = Math.max(ceilingWaitedMs, scan.settle.sinceNavMs);
+                ceilingMutations = Math.max(ceilingMutations, scan.settle.mutations);
                 break;
             default:
                 break;
         }
     }
-    return { rulesAssessed: rules.size, quiet, teardown, ceiling };
+    return { rulesAssessed: rules.size, quiet, teardown, ceiling, ceilingWaitedMs, ceilingMutations };
 }
 /*
  * Google's "good" thresholds for the two Core Web Vitals we can measure in a lab.
@@ -366,6 +374,8 @@ export function totals(envelope) {
     const themes = new Set();
     let violations = 0;
     let incomplete = 0;
+    let judgement = 0;
+    let unmeasured = 0;
     let passes = 0;
     let clean = 0;
     for (const scan of envelope.scans) {
@@ -374,9 +384,17 @@ export function totals(envelope) {
             themes.add(scan.context.theme);
         const scanViolations = scan.findings.filter((f) => f.status === 'violation');
         violations += scanViolations.reduce((sum, f) => sum + f.targets.nodeCount, 0);
-        incomplete += scan.findings
-            .filter((f) => f.status === 'incomplete')
-            .reduce((sum, f) => sum + f.targets.nodeCount, 0);
+        for (const f of scan.findings) {
+            if (f.status !== 'incomplete')
+                continue;
+            incomplete += f.targets.nodeCount;
+            // Unclassified rows count as a judgement: absence is not "measurement",
+            // and inflating our own debt figure would make it useless as a measure.
+            if (f.undecidable === 'measurement')
+                unmeasured += f.targets.nodeCount;
+            else
+                judgement += f.targets.nodeCount;
+        }
         passes += scan.passes.reduce((sum, p) => sum + p.nodeCount, 0);
         if (scanViolations.length === 0)
             clean += 1;
@@ -387,6 +405,8 @@ export function totals(envelope) {
         scans: envelope.scans.length,
         violations,
         incomplete,
+        judgement,
+        unmeasured,
         passes,
         clean,
     };
@@ -448,14 +468,31 @@ export function renderText(input, columns) {
      * strip before a word is read.
      */
     /**
-     * Where the run's own artefacts are, and — usually — the closing block.
+     * Where the run's own artefacts are, and the closing block.
+     *
+     * There used to be a five-line pitch here, on by default, switchable off with
+     * `promo: false`. It is gone, and the reasoning is worth keeping because it
+     * will be proposed again.
+     *
+     * The design answered two questions at once and could not be right about both.
+     * It read as an advert — a product name, a benefit list, a URL — but every
+     * line of it was TRUE ABOUT THEIR RUN, and it was switchable, which only makes
+     * sense for an advert. A disclosure is not optional; you cannot turn off a
+     * limitation.
      *
-     * `promo` is a parameter because of one case: a run that scanned nothing must
-     * not be followed by a pitch. The numbers it would quote are all zero, so it
-     * would read as an advert bolted onto a failure, and the closing block's whole
-     * licence to exist is that it is derived from the run's own figures.
+     * This repository's own rule about `doctor`'s exit code settled it: *we do not
+     * fail a customer's build, and the promise is weaker if it has an opt-out*.
+     * "We do not advertise in your build output" is a stronger promise than "we
+     * advertise unless you ask us not to" — and the flag was a fiction anyway,
+     * governing one of three places the pitch appeared while `verify` and the
+     * SARIF caveat carried it regardless, the second of those into the customer's
+     * own code-scanning UI.
+     *
+     * What survives is the half that was a fact about their run, as a caveat
+     * rather than a pitch: no product name, no URL, and not switchable, because
+     * "this run only" is a real limit on what the reader is holding.
      */
-    const trailer = (promo = true) => [
+    const trailer = (scanned = true) => [
         ...(input.warnings.length > 0 ? [''] : []),
         ...warningLines(input.warnings, width, evidenceQuality(envelope), envelope),
         '',
@@ -465,7 +502,21 @@ export function renderText(input, columns) {
             ? [figureLine(style, '', `${input.htmlReportPath}  ${style.dim('(open in a browser)')}`)]
             : []),
         ...(input.screensPath
-            ? [figureLine(style, '', `${input.screensPath}  ${style.dim('(screenshots, one page)')}`)]
+            ? [
+                figureLine(style, '', `${input.screensPath}  ${style.dim('(screenshots, one page)')}`),
+                /*
+                 * Said where the path is, because that is where someone decides
+                 * whether to keep it — and measured rather than assumed: Playwright
+                 * clears its whole output directory at the start of a run, including
+                 * captures belonging to tests that run does not execute.
+                 *
+                 * The sheet indexes those images rather than copying them, which is a
+                 * privacy guarantee and not tidiness: Playwright's own captures are
+                 * unmasked, and this package writes no images at all.
+                 */
+                ...continuationLines(style, "That page links to Playwright's images rather than copying them, and Playwright " +
+                    'clears its output directory on its next run — so copy it now if this run matters.', width),
+            ]
             : []),
         ...(input.reportPath
             ? [figureLine(style, '', `${input.reportPath}  ${style.dim('(Playwright report)')}`)]
@@ -477,7 +528,13 @@ export function renderText(input, columns) {
                 ...groupLines(input.group).flatMap((line) => wrap(line, Math.max(40, width - 8), '').map((wrapped) => figureLine(style, '', wrapped))),
             ]
             : []),
-        ...(promo ? promoLines(input, count, width) : ['']),
+        /*
+         * A run that scanned nothing gets no caveat, for the reason the pitch used
+         * to be suppressed there: the figures are all zero, and a line about what
+         * this run does not cover reads as noise bolted onto a failure. The block
+         * above already says nothing was looked at, which is the stronger statement.
+         */
+        ...(scanned ? historyCaveat(input, style, width) : ['']),
     ];
     lines.push('');
     lines.push(`  ${style.accent('Setsquare')} — accessibility evidence`);
@@ -533,8 +590,27 @@ export function renderText(input, columns) {
          */
         lines.push(heading(style, 'Accessibility'));
         lines.push(figureLine(style, 0, 'surfaces scanned'));
-        lines.push(...continuationLines(style, 'This is not a clean result — it means nothing was looked at. Usually the ' +
-            'suite never opened a page, or every test finished before its page settled.', width));
+        /*
+         * Name the cause when the run knows it, and only then.
+         *
+         * "Usually the suite never opened a page, or every test finished before its
+         * page settled" is a guess offered in place of a fact the envelope is
+         * holding. On a lane run with `SETSQUARE_AUTOSCAN=0` both halves of that
+         * sentence are false and both describe a broken suite, which is how a
+         * perfectly healthy functional lane came to read as a failure.
+         *
+         * "This is not a clean result" stays either way. Nothing was looked at, and
+         * that is true whether or not the reason is reassuring.
+         */
+        const offRows = (envelope.unscanned ?? []).filter((row) => row.reason === 'autoscan-disabled');
+        const allOff = offRows.length > 0 && offRows.length === (envelope.unscanned ?? []).length;
+        lines.push(...continuationLines(style, allOff
+            ? 'This is not a clean result — it means nothing was looked at. Automatic ' +
+                'scanning was off for this run (SETSQUARE_AUTOSCAN=0), so the surfaces ' +
+                'below were visited and never scanned. That is the configuration working ' +
+                'as asked; it is not a fault in the suite.'
+            : 'This is not a clean result — it means nothing was looked at. Usually the ' +
+                'suite never opened a page, or every test finished before its page settled.', width));
         lines.push(...trailer(false));
         return join(lines);
     }
@@ -581,10 +657,54 @@ export function renderText(input, columns) {
     lines.push(figureLine(style, failingRules, `rule${failingRules === 1 ? '' : 's'} failing, on ${n(count.violations)} ` +
         `element${count.violations === 1 ? '' : 's'}`));
     if (count.incomplete > 0) {
-        lines.push(figureLine(style, count.incomplete, 'needing review — axe could not decide'));
+        /*
+         * Two figures, because two different things were sharing one word — and the
+         * old line credited axe for both, which stopped being true when most of the
+         * undecided rows came from our own checks.
+         */
+        if (count.judgement > 0) {
+            lines.push(figureLine(style, count.judgement, 'needing a judgement — a person decides'));
+        }
+        if (count.unmeasured > 0) {
+            lines.push(figureLine(style, count.unmeasured, 'we could not measure — our debt, not yours'));
+        }
     }
     lines.push(figureLine(style, count.passes, style.dim(`checks passed, across ${quality.rulesAssessed} distinct rules`)));
+    /*
+     * The element figure holds still only in the sense that the rule count above it
+     * does. Said once, here, where someone is about to quote a number.
+     */
+    if (count.violations > 0) {
+        lines.push(...continuationLines(style, 'Rule and surface counts are the stable pair. Element counts follow how much data ' +
+            'is on the page, so they move between runs of unchanged code — measured at 1,712 ' +
+            'and then 1,626 on two runs of one commit, while rules × surfaces held at 230.', width));
+    }
     lines.push(...conformanceLines(input, style, width));
+    /*
+     * THE CUT. Everything below is the report's job unless asked for.
+     *
+     * Friction record run 4 measured 1,080 lines on Eos, and said the useful part
+     * was the first forty. Both halves of that matter: a summary nobody reads is
+     * not merely long, it takes the part that WAS worth reading down with it,
+     * because by the end of a run those lines have scrolled away.
+     *
+     * Measured on the fixture suite at 26 surfaces — where the enumerations are
+     * the bulk, and the criteria table is the last thing a person reads:
+     *
+     *   RULES FAILING 152 · NEEDS A JUDGEMENT 82 · CONFORMANCE 62 · SURFACES 36
+     *
+     * What still prints below the cut is anything that QUALIFIES the figures
+     * above — what could not be checked, what is worth knowing. Dropping those
+     * would make the short summary a MORE confident one than the long summary,
+     * which is exactly the wrong direction for an evidence product: the reader
+     * would be left with the numbers and without the reasons to distrust them.
+     */
+    if (!input.verbose) {
+        lines.push(...qualifyingBlocks(input, style, width));
+        lines.push(...moreInTheReport(input, style, width));
+        lines.push(...trailer());
+        return join(lines);
+    }
     lines.push(...gridLines(input, style, width));
     lines.push('');
     /*
@@ -711,24 +831,46 @@ export function renderText(input, columns) {
         for (const group of failingRuleGroups)
             lines.push(...ruleEntry(group));
     }
-    if (review.length > 0) {
+    /*
+     * TWO sections, because two different things were sharing one word.
+     *
+     * Friction record run 4 measured what that cost: roughly 273 of Eos's 374
+     * needs-review rows were measurements we had not taken rather than judgements
+     * a person must make. A reader asked to work through a bucket where most rows
+     * cannot be acted on without a colour picker learns to skip the bucket — and
+     * the rows they skip include the ones an auditor actually wants.
+     *
+     * So the judgements come first and keep the section a person is meant to read,
+     * and our own debt is stated separately, as a figure, under our own name.
+     */
+    const judgement = review.filter((group) => group.undecidable !== 'measurement');
+    const measurement = review.filter((group) => group.undecidable === 'measurement');
+    if (judgement.length > 0) {
         /*
-         * A section of its own, not a parenthesis on the failure list.
-         *
-         * `incomplete` is axe saying "I could not decide" — most often contrast over
-         * a background image or a gradient, where the ratio is genuinely
-         * uncomputable from the DOM. These are neither passes nor failures, and
-         * listing them beside violations invites both of the available mistakes:
-         * treating them as broken and panicking, or treating them as fine and
-         * shipping. They are the one category that REQUIRES a person, which is worth
-         * saying plainly given we sell evidence.
+         * A section of its own, not a parenthesis on the failure list. These are
+         * neither passes nor failures, and listing them beside violations invites
+         * both available mistakes: treating them as broken and panicking, or
+         * treating them as fine and shipping.
          */
         lines.push('');
-        lines.push(heading(style, 'Needs review'));
-        lines.push(...continuationLines(style, 'axe could not decide these by itself — usually contrast over an image or a ' +
-            'gradient. Not passes and not failures; a person has to look.', width));
+        lines.push(heading(style, 'Needs a judgement'));
+        lines.push(...continuationLines(style, 'Measured as far as measuring goes, and what is left is not mechanical — ' +
+            'does this heading describe what it heads, is this error announced. A person ' +
+            'settles these by looking, with no tools.', width));
+        lines.push('');
+        for (const group of judgement)
+            lines.push(...ruleEntry(group));
+    }
+    if (measurement.length > 0) {
+        const elements = measurement.reduce((sum, group) => sum + group.nodes, 0);
+        lines.push('');
+        lines.push(heading(style, 'Could not measure'));
+        lines.push(...continuationLines(style, `${elements} element${elements === 1 ? '' : 's'} where we did not get a number — a ` +
+            'gradient with no single colour behind it, a value we could not resolve. ' +
+            'This is our debt rather than your work, and it is here so it can be seen ' +
+            'falling rather than sitting inside a bucket labelled as yours.', width));
         lines.push('');
-        for (const group of review)
+        for (const group of measurement)
             lines.push(...ruleEntry(group));
     }
     /*
@@ -755,6 +897,52 @@ export function renderText(input, columns) {
     lines.push(...trailer());
     return join(lines);
 }
+/**
+ * The blocks that qualify the figures, and print above the cut as well as below.
+ *
+ * Deliberately a short list rather than "the insight blocks". `Could not be
+ * checked` is the surfaces the run reached and could not scan, and `Worth
+ * knowing` carries the scans the settle ceiling cut short — both are reasons to
+ * trust the numbers above LESS, and a concise summary that dropped them would be
+ * more confident than the full one. `Page runtime`, `Across surfaces`,
+ * `Coverage` and `Suite timing` are additional findings rather than
+ * qualifications, so they wait for the report or `--verbose`.
+ */
+const QUALIFYING = new Set(['Could not be checked', 'Worth knowing']);
+function qualifyingBlocks(input, style, width) {
+    const lines = [];
+    for (const block of insightBlocks(input.envelope, {
+        ...(input.declaredAxes ? { declaredAxes: input.declaredAxes } : {}),
+        ...(input.unmeasuredAxes ? { unmeasuredAxes: input.unmeasuredAxes } : {}),
+        ...(input.suite ? { suite: input.suite } : {}),
+        ...(input.overhead ? { overhead: input.overhead } : {}),
+    })) {
+        if (!QUALIFYING.has(block.heading))
+            continue;
+        lines.push(...figureBlock(style, block.heading, block.figures, block.notes, width));
+    }
+    return lines;
+}
+/**
+ * One line naming what was left out, and how to see it.
+ *
+ * "See the report" on its own is a dead end — a reader cannot tell whether the
+ * thing they want is in there. So the sections are named, and so is the way to
+ * get them back in a terminal, because a CI log is a place where scrollback IS
+ * the artifact and there is no report to open.
+ */
+function moreInTheReport(input, style, width) {
+    const count = totals(input.envelope);
+    const rules = new Set(input.envelope.scans.flatMap((scan) => scan.findings.filter((f) => f.status === 'violation').map((f) => f.ruleId))).size;
+    if (rules === 0 && count.incomplete === 0)
+        return [];
+    return [
+        '',
+        ...continuationLines(style, `Which rules, on which surfaces, with the selectors and the criteria grid: in the ` +
+            `report below. SETSQUARE_VERBOSE=1 prints all of it here instead, for a CI log with ` +
+            `no report to open.`, width),
+    ];
+}
 /**
  * Join the assembled lines, collapsing any run of blank lines to one.
  *
@@ -996,7 +1184,19 @@ function conformanceLines(input, style, width) {
         lines.push(figureLine(style, rulesPassed, 'criteria: every automated rule passed — evidence, not conformance'));
     }
     if (cantTell > 0) {
-        lines.push(figureLine(style, cantTell, 'criteria need a person — automation could not decide'));
+        /*
+         * A criterion, not a row — so it stays one state.
+         *
+         * A cell rolls up many findings and can hold both kinds at once, and there is
+         * no defensible precedence between them: a criterion with one unmeasured row
+         * and one genuine judgement is not more one than the other. "Automation could
+         * not decide" is true of the cell whichever it holds, and the two sections
+         * above carry the why at the level where a row is the unit.
+         *
+         * The old wording — "criteria need a person" — was the one that stopped being
+         * true, because a cell can now be undecided entirely because of our debt.
+         */
+        lines.push(figureLine(style, cantTell, 'criteria automation could not decide — see the two sections above'));
     }
     if (inapplicable > 0) {
         // EARL's inapplicable, spelled out: the checks RAN and the pages contain
@@ -1159,7 +1359,7 @@ function gridLines(input, style, width) {
         lines.push((indent + label + cells).trimEnd());
     }
     lines.push('');
-    lines.push(...continuationLines(style, `${glyph.failed} failed   ${glyph.cantTell} needs a person   ` +
+    lines.push(...continuationLines(style, `${glyph.failed} failed   ${glyph.cantTell} automation could not decide   ` +
         `${glyph.rulesPassed} every automated rule passed   ` +
         `${glyph.inapplicable} nothing to check here   ` +
         `${glyph.assertedBySuite} covered by your own test   ` +
@@ -1198,9 +1398,28 @@ function warningLines(warnings, width, quality, envelope) {
      * dying in the envelope.
      */
     if (quality && quality.ceiling > 0) {
-        notes.push(`${quality.ceiling} scan${quality.ceiling === 1 ? ' was' : 's were'} cut short after ` +
-            'waiting 5s for the page to settle, so they describe a page that was still ' +
-            'changing. Treat a clean result from those as weaker evidence.');
+        /*
+         * Say WHOSE window it was, and WHAT the page was doing in it.
+         *
+         * The old sentence said "after waiting 5s", which is the default — a suite
+         * that raised `settleTimeoutMs` was quoted a number it had changed. And it
+         * described "a page that was still changing" without the one figure that
+         * tells a reader which of the two available actions is theirs: a page a few
+         * mutations from quiet settles if given longer, and a page with a poller
+         * never will, and no timeout helps the second.
+         *
+         * Both numbers are measured off the scans themselves rather than read from
+         * configuration, which also means they stay true if the window is changed
+         * per project.
+         */
+        const waited = `${(quality.ceilingWaitedMs / 1000).toFixed(1)}s`;
+        notes.push(`${quality.ceiling} scan${quality.ceiling === 1 ? ' was' : 's were'} cut short by our ` +
+            `settle window — the longest waited ${waited} and the page was still changing, with ` +
+            `${quality.ceilingMutations.toLocaleString('en-GB')} DOM mutation` +
+            `${quality.ceilingMutations === 1 ? '' : 's'} counted since it loaded. Treat a clean ` +
+            'result from those as weaker evidence. If the page settles and simply needs longer, ' +
+            'raise `settleTimeoutMs`; if something polls or animates forever, no window will help ' +
+            'and the scan is of a moving page by nature.');
     }
     if (warnings.length === 0 && notes.length === 0)
         return [];
@@ -1232,22 +1451,29 @@ function warningLines(warnings, width, quality, envelope) {
  * only things the free layer structurally cannot do. Nothing is removed from the
  * local tool to create the paid one, and this block must never imply otherwise.
  */
-function promoLines(input, count, width) {
-    if (!input.promo)
-        return [];
+/**
+ * What this run is, and what it is not, in one line.
+ *
+ * Derived from the run's own numbers, which is the only thing that licensed the
+ * block it replaces. No product name and no URL: the reader is running the tool,
+ * so they know where it comes from, and a limit stated as a limit does not need
+ * a destination attached to be useful.
+ *
+ * Not switchable. "Local runs keep no history, baselines or comparison" is a
+ * fact about the artefact in front of them — the dev plan holds it as a
+ * permanent property of the free layer rather than a gap — so it belongs beside
+ * the figures whether or not anyone would prefer it quiet.
+ */
+function historyCaveat(input, style, width) {
+    const count = totals(input.envelope);
     const failingRules = new Set(input.envelope.scans.flatMap((scan) => scan.findings.filter((f) => f.status === 'violation').map((f) => f.ruleId))).size;
     const subject = failingRules > 0
         ? `${failingRules} rule${failingRules === 1 ? '' : 's'} failing across ${count.surfaces} surface${count.surfaces === 1 ? '' : 's'}`
         : `${count.surfaces} clean surface${count.surfaces === 1 ? '' : 's'}`;
     return [
         '',
-        // Wrapped like everything else: a block that overflows the terminal is a
-        // block that looks broken, and this one is read on every single run.
-        ...wrap(`${subject}, this run only. Setsquare Cloud adds what a local tool cannot:`, width - 4, '').map((line) => `  ${line}`),
-        '    · is this better or worse than last week?  (local runs keep no history)',
-        '    · which change introduced each one, on the PR that did it',
-        '    · the root cause behind repeats, grouped instead of repeated',
-        `    ${PLATFORM_URL}`,
+        ...continuationLines(style, `${subject}, this run only — local runs keep no history, no baselines and no ` +
+            'comparison with a previous run, by design.', width),
         '',
     ];
 }