@fulldotdev/scan 0.1.0 → 0.3.0

package/README.md

@@ -26,7 +26,7 @@ Options:
 | Flag | Meaning |
 | --- | --- |
 | `--out <dir>` | Output directory, default `./scan` |
-| `--quick` | Crawl only: no browser pass, no Lighthouse |
+| `--quick` | The daily scan: crawl only, no browser pass, Lighthouse or external sources |
 | `--deep` | Deeper measurement: rendering observation and Lighthouse repeats |
 | `--lighthouse <mode>` | `templates` (default: homepage plus one rotating page per template), `all`, or `off` |
 | `--no-browser` | Skip the browser pass and Lighthouse |
@@ -48,9 +48,16 @@ Local targets (localhost, private addresses, any port) are allowed automatically
 
 `GOOGLE_API_KEY` (a Google Cloud key with the Chrome UX Report API and Safe Browsing API enabled) enables real-user vitals and the Safe Browsing check.
 
+## In a repository
+
+Two rhythms: a daily crawl (`--quick`, seconds to minutes) and a weekly full scan (browser pass on every page, Lighthouse per template, external sources). The workflow in [`templates/github-workflow.yml`](templates/github-workflow.yml) does both, scans every pull request on its Netlify deploy preview, comments the result on the pull request, fails on critical findings and pushes scans of the live site to scan.full.dev. Copy it to `.github/workflows/scan.yml`, set `SITE_URL`, and add the secret `FULLSCAN_KEY`.
+
+For an agent working on a site: run `fullscan http://localhost:3000`, read `scan/report.md`, change the site, and run again with `--previous scan/result.json` until nothing is left.
+
 ## Output
 
 - `result.json`: `scan` (options, coverage, status), `report` (overview, findings per topic, checks with outcome, coverage, changes), `findings` (every finding with all affected items), `pages` (one compact row per page: status, type, language, indexable, depth, word count, template, Lighthouse scores, finding counts), `assets` (every file and external destination with status, size and the pages that use it).
+- `summary.md`: counts, changes and the most important problems, short enough for a pull request comment.
 - `report.md`: the same report as readable Markdown, made for people and language models.
 - `records.ndjson`: every raw observation (page extraction, rendered comparison, axe results, Lighthouse evidence, DNS, TLS, sitemaps and so on).
 - `artifacts/`: screenshots, retrieved sitemaps, robots.txt and agent files, with `index.json`.

package/SKILL.md

@@ -0,0 +1,27 @@
+---
+name: fullscan
+description: Scan a website or a local dev server for problems with @fulldotdev/scan and read the result. Use when asked to check, audit or scan a site, or to verify a change did not introduce problems.
+---
+
+# fullscan
+
+Run it against a live URL or the local dev server:
+
+```sh
+npx @fulldotdev/scan http://localhost:3000 --quick --out scan
+npx @fulldotdev/scan https://example.com --out scan
+```
+
+`--quick` crawls every page without a browser (fast, seconds). Without it the run also renders every page, runs axe-core, and runs Lighthouse on one page per template (minutes).
+
+With an account on the platform, `--push` stores the result there as well:
+
+```sh
+npx @fulldotdev/scan https://example.com --push --key $FULLSCAN_KEY
+```
+
+It prints the site and run id it stored, and creates the site on the first push.
+
+Read `scan/report.md` first. It lists the problems per topic with severity, confidence, problem, impact and evidence. For the full list of affected pages per finding read `findings[]` in `scan/result.json`; for per-page rows read `pages[]`; for files and external destinations read `assets[]`. `report.checks[]` tells which rules ran and which did not.
+
+The tool reports problems and their impact only. Deciding what, if anything, to change is up to you and the person you work for. Run it again with `--previous scan/result.json` after a change to see what is new, resolved or unverified.

package/dist/cli.js

@@ -16,7 +16,7 @@ per template, and every problem found is written as JSON and Markdown.
 
 Options:
   --out <dir>            Output directory (default: ./scan)
-  --quick                Crawl only: no browser pass, no Lighthouse
+  --quick                The daily scan: crawl only, no browser, Lighthouse or external sources
   --deep                 Deeper measurement: rendering observation, Lighthouse repeats
   --lighthouse <mode>    templates (default), all, or off
   --no-browser           Skip the browser pass and Lighthouse
@@ -29,6 +29,8 @@ Options:
   --push                 Push the result to the platform after the scan
   --key <key>            API key for --push (default: FULLSCAN_KEY)
   --platform <url>       Platform to push to (default: https://scan.full.dev)
+  --run <id>             Fill this queued run of the platform with the result
+  --scheduled            Store the result as a run of the nightly schedule
   --json                 Print the result JSON to stdout instead of progress
   --quiet                No progress output
   -h, --help             Show this help
@@ -53,6 +55,8 @@ const { values, positionals } = parseArgs({
         push: { type: "boolean", default: false },
         key: { type: "string" },
         platform: { type: "string", default: "https://scan.full.dev" },
+        run: { type: "string" },
+        scheduled: { type: "boolean", default: false },
         json: { type: "boolean", default: false },
         quiet: { type: "boolean", default: false },
         help: { type: "boolean", short: "h", default: false },
@@ -76,11 +80,45 @@ const options = {
         ? "off"
         : (values.lighthouse ??
             "templates"),
-    external: values.external,
+    external: values.external && !values.quick,
     allowLocal: local,
     ...(values.pages ? { maxPages: Number(values.pages) } : {}),
 };
+const key = values.key ?? process.env.FULLSCAN_KEY;
+const platform = values.platform.replace(/\/+$/, "");
+// One call to the platform's typed API. Returns the result or throws.
+async function callPlatform(path, operation, args) {
+    const response = await fetch(`${platform}/api/${path}`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${key}`,
+        },
+        body: JSON.stringify({ operation, args }),
+    });
+    const payload = (await response.json().catch(() => null));
+    if (!response.ok)
+        throw new Error(payload?.error?.message ?? `${response.status} ${response.statusText}`);
+    return payload?.result;
+}
 let previous;
+let previousRotation;
+// With a key and no file, the platform tells the scan what the site looked
+// like last time, so changes are tracked on a runner just as well.
+if (!values.previous && key) {
+    try {
+        const result = await callPlatform("query", "runs.previous", { url });
+        if (result) {
+            const { rotation: stored, ...rest } = result;
+            previous = rest;
+            if (typeof stored === "number")
+                previousRotation = stored;
+        }
+    }
+    catch (error) {
+        process.stderr.write(`Could not read the previous scan: ${error instanceof Error ? error.message : String(error)}\n`);
+    }
+}
 if (values.previous) {
     const file = JSON.parse(await readFile(values.previous, "utf8"));
     previous = {
@@ -102,10 +140,12 @@ if (values.previous) {
 }
 const rotation = values.rotation
     ? Number(values.rotation)
-    : previous
-        ? Number(JSON.parse(await readFile(values.previous, "utf8")).scan?.rotation ??
-            -1) + 1
-        : 0;
+    : previousRotation !== undefined
+        ? previousRotation
+        : values.previous
+            ? Number(JSON.parse(await readFile(values.previous, "utf8")).scan?.rotation ??
+                -1) + 1
+            : 0;
 const quiet = values.quiet || values.json;
 const log = (line) => {
     if (!quiet)
@@ -154,6 +194,7 @@ const out = values.out;
 await mkdir(join(out, "artifacts"), { recursive: true });
 await writeFile(join(out, "result.json"), JSON.stringify(result, null, 2));
 await writeFile(join(out, "report.md"), formatReport(report) + "\n");
+await writeFile(join(out, "summary.md"), summary() + "\n");
 await writeFile(join(out, "records.ndjson"), records.map((r) => JSON.stringify(r)).join("\n") + "\n");
 const index = {};
 for (const artifact of state.artifacts.values()) {
@@ -166,41 +207,100 @@ for (const artifact of state.artifacts.values()) {
     };
 }
 await writeFile(join(out, "artifacts", "index.json"), JSON.stringify(index, null, 2));
-// The push carries the result only: records and artifacts stay local.
+// The push carries the result; the observations and the screenshots follow
+// it in their own calls, so the run on the platform is as complete as one
+// the platform ran itself.
 let pushed;
 if (values.push) {
-    const key = values.key ?? process.env.FULLSCAN_KEY;
     if (!key) {
         process.stderr.write("A key is required to push: pass --key or set FULLSCAN_KEY.\n");
         process.exit(1);
     }
-    const platform = values.platform.replace(/\/+$/, "");
-    const response = await fetch(`${platform}/api/command`, {
-        method: "POST",
-        headers: {
-            "Content-Type": "application/json",
-            Authorization: `Bearer ${key}`,
-        },
-        body: JSON.stringify({
-            operation: "runs.push",
-            args: {
-                url: scan.url,
-                scan,
-                report: result.report,
-                findings,
-                pages,
-                assets,
-            },
-        }),
-    });
-    const payload = (await response.json().catch(() => null));
-    if (!response.ok || !payload?.result) {
-        process.stderr.write(`Push failed: ${payload?.error?.message ?? `${response.status} ${response.statusText}`}\n`);
+    try {
+        pushed = await callPlatform("command", "runs.push", {
+            url: scan.url,
+            scan,
+            report: result.report,
+            findings,
+            pages,
+            assets,
+            ...(values.run ? { runId: values.run } : {}),
+            ...(values.scheduled ? { scheduled: true } : {}),
+        });
+    }
+    catch (error) {
+        process.stderr.write(`Push failed: ${error instanceof Error ? error.message : String(error)}\n`);
+        process.exit(1);
+    }
+    if (!pushed) {
+        process.stderr.write("Push failed: the platform stored no run.\n");
         process.exit(1);
     }
-    pushed = payload.result;
     if (values.json)
         process.stderr.write(`Pushed to ${platform}: site ${pushed.siteId}, run ${pushed.runId}\n`);
+    await pushRecords(pushed.runId);
+    await pushScreenshots(pushed.runId);
+}
+// Observations go up in batches that stay under 3 MB of JSON, because the
+// platform's request limit is a few megabytes. A batch that fails costs the
+// page detail of this run, never the run itself.
+async function pushRecords(runId) {
+    const limit = 3 * 1024 * 1024;
+    const batches = [[]];
+    let size = 0;
+    for (const record of records) {
+        const bytes = Buffer.byteLength(JSON.stringify(record));
+        if (size + bytes > limit && batches[batches.length - 1].length) {
+            batches.push([]);
+            size = 0;
+        }
+        batches[batches.length - 1].push(record);
+        size += bytes;
+    }
+    for (const [index, batch] of batches.entries()) {
+        const final = index === batches.length - 1;
+        try {
+            await callPlatform("command", "runs.pushRecords", {
+                runId,
+                records: batch,
+                final,
+            });
+        }
+        catch (error) {
+            process.stderr.write(`Could not push ${batch.length} observations: ${error instanceof Error ? error.message : String(error)}\n`);
+        }
+    }
+}
+// Screenshots are raw bytes, so they go to their own endpoint instead of
+// swelling the JSON push by a third.
+async function pushScreenshots(runId) {
+    for (const artifact of state.artifacts.values()) {
+        if (artifact.kind !== "screenshot")
+            continue;
+        try {
+            const query = new URLSearchParams({
+                runId,
+                kind: "screenshot",
+                key: artifact.key,
+            });
+            const response = await fetch(`${platform}/api/artifacts/upload?${query.toString()}`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": artifact.contentType,
+                    Authorization: `Bearer ${key}`,
+                },
+                body: new Uint8Array(artifact.body),
+            });
+            if (!response.ok) {
+                const payload = (await response.json().catch(() => null));
+                throw new Error(payload?.error?.message ??
+                    `${response.status} ${response.statusText}`);
+            }
+        }
+        catch (error) {
+            process.stderr.write(`Could not upload the screenshot of ${artifact.key}: ${error instanceof Error ? error.message : String(error)}\n`);
+        }
+    }
 }
 if (values.json)
     process.stdout.write(JSON.stringify(result) + "\n");
@@ -222,6 +322,30 @@ if (values["fail-on"]) {
         : c.critical > 0 || c.warning > 0;
     process.exit(failed ? 1 : 0);
 }
+// A short version of the report for a pull request comment or a job summary.
+function summary() {
+    const c = report.counts;
+    const top = report.findings
+        .filter((f) => f.severity !== "hint")
+        .slice(0, 10)
+        .map((f) => `- **${f.title}** (${f.severity}, ${f.count} on ${f.pages} pages). ${f.impact}`);
+    const changes = report.changes
+        ? [
+            "",
+            `Since the previous scan: ${report.changes.newFindings.length} new, ${(report.changes.resolvedFindings ?? []).length} resolved, ${(report.changes.worsenedFindings ?? []).length} worse.`,
+            ...report.changes.newFindings
+                .slice(0, 10)
+                .map((f) => `- New: **${f.title}** (${f.severity}, ${f.count})`),
+        ]
+        : [];
+    return [
+        `### Scan of ${new URL(scan.url).hostname}`,
+        "",
+        `${report.pages.observed} pages, **${c.critical} critical**, **${c.warning} warnings**, ${c.hint ?? 0} hints. Status ${scan.status}.`,
+        ...changes,
+        ...(top.length ? ["", "Most important problems:", ...top] : []),
+    ].join("\n");
+}
 function extension(contentType) {
     if (/png/.test(contentType))
         return ".png";

package/dist/engine/browser.js

@@ -24,6 +24,8 @@ export async function openBrowserSession() {
         ];
         const executablePath = env("CHROME_PATH") || localPaths.find(existsSync) || undefined;
         const args = [
+            // Extra flags for environments that need them (a serverless Chrome).
+            ...JSON.parse(env("CHROME_ARGS") || "[]"),
             "--no-sandbox",
             "--disable-dev-shm-usage",
             `--proxy-server=http://127.0.0.1:${proxy.port}`,
@@ -80,8 +82,7 @@ const axeTags = [
 export async function collectBrowser(scan, url, http, session) {
     await resolvePublic(new URL(url).hostname);
     const robots = await http.getRobots(url);
-    if (!robots.allowed ||
-        robots.parser?.isAllowed(url, "FulldevScan") === false)
+    if (!robots.allowed || robots.parser?.isAllowed(url, "FulldevScan") === false)
         return {
             observations: [
                 {
@@ -93,6 +94,9 @@ export async function collectBrowser(scan, url, http, session) {
         };
     const { proxy, browser, chromeVersion } = session;
     const blockedBefore = proxy.blocked.length;
+    // Hosts this page asked for, so blocked requests of a tab rendering at
+    // the same time are not counted here.
+    const requestedHosts = new Set();
     const result = { observations: [], artifacts: [], candidates: [] };
     const errors = [];
     let page;
@@ -102,8 +106,9 @@ export async function collectBrowser(scan, url, http, session) {
             stage: "budget",
             error: `Browser job exceeded ${budgetSeconds} seconds`,
         });
-        // Closing the browser aborts the page; the worker reopens a session.
-        void browser.close();
+        // Closing the tab aborts this page and leaves its neighbours alone; a
+        // tab that will not close takes the browser with it.
+        void (page ? page.close().catch(() => browser.close()) : browser.close());
     }, budgetSeconds * 1000);
     watchdog.unref();
     try {
@@ -134,6 +139,12 @@ export async function collectBrowser(scan, url, http, session) {
         const cdp = await page.createCDPSession();
         await cdp.send("Network.enable");
         cdp.on("Network.requestWillBeSent", (e) => {
+            try {
+                requestedHosts.add(new URL(e.request?.url ?? "").hostname);
+            }
+            catch {
+                // Not a URL with a host (data:, blob:).
+            }
             if (staticTypes.test(e.type) && /^https?:/.test(e.request?.url ?? ""))
                 staticResources.add(e.request.url);
         });
@@ -622,7 +633,9 @@ export async function collectBrowser(scan, url, http, session) {
         data: {
             status: errors.length ? "partial" : "completed",
             errors,
-            blockedNetwork: proxy.blocked.slice(blockedBefore),
+            blockedNetwork: proxy.blocked
+                .slice(blockedBefore)
+                .filter((entry) => [...requestedHosts].some((host) => entry.target.includes(host))),
         },
     });
     return result;

package/dist/engine/proxy.js

@@ -7,6 +7,8 @@ export async function createEgressProxy() {
     const sockets = new Set();
     const remember = (socket) => {
         sockets.add(socket);
+        // Several checks share a kept-alive socket; each adds its own listeners.
+        socket.setMaxListeners(50);
         socket.on("close", () => sockets.delete(socket));
         socket.setTimeout(30000, () => socket.destroy());
     };

package/dist/engine/run.d.ts

@@ -1,4 +1,4 @@
-import { ScanState } from "./state.js";
+import { ScanState, type ScanSnapshot } from "./state.js";
 import { type JobKind, type Options, type Scan } from "./types.js";
 import { type PreviousScan, type Report, type StoredFinding } from "../report/build.js";
 export type ScanEvent = {
@@ -26,6 +26,10 @@ export interface RunInput {
     rotation?: number;
     previous?: PreviousScan;
     budgetMs?: number;
+    resume?: ScanSnapshot;
+    pauseAtBudget?: boolean;
+    checkpointEveryMs?: number;
+    onCheckpoint?: (snapshot: ScanSnapshot) => Promise<void>;
     signal?: AbortSignal;
     onEvent?: (event: ScanEvent) => void;
 }
@@ -36,3 +40,6 @@ export interface RunOutput {
     state: ScanState;
 }
 export declare function runScan(input: RunInput): Promise<RunOutput>;
+export declare function runScanStep(input: RunInput): Promise<RunOutput | {
+    paused: ScanSnapshot;
+}>;

package/dist/engine/run.js

@@ -16,11 +16,21 @@ const crawlKinds = [
 ];
 const maxJobMs = 300000;
 export async function runScan(input) {
-    const options = optionsSchema.parse(input.options ?? {});
+    const result = await runScanStep({ ...input, pauseAtBudget: false });
+    if ("paused" in result)
+        throw new Error("Scan paused unexpectedly");
+    return result;
+}
+// One stretch of a scan: runs until it is done or, with `pauseAtBudget`,
+// until the budget runs out, and then hands back a snapshot.
+export async function runScanStep(input) {
+    const options = input.resume
+        ? input.resume.scan.options
+        : optionsSchema.parse(input.options ?? {});
     setLocalTargetsAllowed(options.allowLocal);
-    const url = normalizeUrl(input.url);
+    const url = input.resume ? input.resume.scan.url : normalizeUrl(input.url);
     await resolvePublic(new URL(url).hostname);
-    const scan = {
+    const scan = input.resume?.scan ?? {
         id: randomUUID(),
         url,
         status: "running",
@@ -29,20 +39,33 @@ export async function runScan(input) {
         rotation: input.rotation ?? 0,
         coverage: { omittedCandidates: 0 },
     };
-    const state = new ScanState(scan);
+    const state = input.resume
+        ? ScanState.restore(input.resume)
+        : new ScanState(scan);
+    let phase = input.resume?.phase ?? "crawl";
     const emit = (event) => input.onEvent?.(event);
     emit({ type: "start", scan });
     const session = new HttpSession();
     const http = new HttpClient(options, undefined, undefined, session);
     const deadline = input.budgetMs ? Date.now() + input.budgetMs : Infinity;
     let browser;
+    let opening;
+    // One Chrome for the scan. Jobs that ask at the same moment share the
+    // launch instead of starting two browsers.
     const browserSession = async () => {
-        const { openBrowserSession } = await import("./browser.js");
-        if (browser && !browser.connected) {
-            await browser.close();
-            browser = undefined;
-        }
-        return (browser ??= await openBrowserSession());
+        if (browser?.connected)
+            return browser;
+        opening ??= (async () => {
+            const { openBrowserSession } = await import("./browser.js");
+            await browser?.close();
+            browser = await openBrowserSession();
+            opening = undefined;
+            return browser;
+        })().catch((error) => {
+            opening = undefined;
+            throw error;
+        });
+        return opening;
     };
     const run = async (job) => {
         switch (job.kind) {
@@ -107,7 +130,8 @@ export async function runScan(input) {
             job.error = message;
             job.status = job.attempts < limit ? "queued" : "failed";
             job.durationMs = Math.round(performance.now() - started);
-            if (job.status === "failed" && ["browser", "lighthouse"].includes(job.kind))
+            if (job.status === "failed" &&
+                ["browser", "lighthouse"].includes(job.kind))
                 state.record({
                     observations: [
                         {
@@ -133,50 +157,98 @@ export async function runScan(input) {
                 pending: state.pending(kinds),
             });
         }
-        // Chromium and Lighthouse leave memory behind; a fresh browser for the
-        // next page is cheaper than the process growing without bound.
-        if (["browser", "lighthouse"].includes(job.kind) &&
-            browser &&
-            process.memoryUsage().rss > 1500 * 1024 * 1024) {
-            await browser.close();
-            browser = undefined;
-        }
     };
+    // How many jobs of a kind run at once. Link and file checks mostly wait on
+    // the network; two pages render side by side as tabs of one Chrome. Pages
+    // of the scanned site are fetched one at a time so the site sets the pace,
+    // and Lighthouse runs alone because a busy CPU worsens its scores.
+    const concurrency = {
+        resource: 6,
+        browser: 2,
+    };
+    const browserKinds = ["browser", "lighthouse"];
+    let lastCheckpoint = Date.now();
+    // True when the budget ran out and the scan should pause here.
     const drain = async (kinds) => {
-        let job;
-        while ((job = state.next(kinds))) {
-            if (Date.now() > deadline || input.signal?.aborted) {
+        const inFlight = new Map();
+        const running = (list) => [...inFlight.keys()].filter((j) => list.includes(j.kind)).length;
+        let recycle = false;
+        for (;;) {
+            const stopping = Date.now() > deadline || !!input.signal?.aborted;
+            let job;
+            while (!stopping &&
+                (job = state.next(kinds.filter((kind) => running([kind]) < (concurrency[kind] ?? 1) &&
+                    // No new tab while Chrome waits to be replaced.
+                    !(recycle && browserKinds.includes(kind)))))) {
+                const current = job;
+                current.status = "running";
+                inFlight.set(current, execute(current, kinds).finally(() => inFlight.delete(current)));
+            }
+            if (!inFlight.size) {
+                if (!stopping || !state.pending(kinds))
+                    break;
+                if (input.pauseAtBudget)
+                    return true;
                 state.cancelQueued();
                 scan.coverage.expired = true;
                 break;
             }
-            await execute(job, kinds);
+            await Promise.race(inFlight.values());
+            // Chromium and Lighthouse leave memory behind; a fresh browser is
+            // cheaper than the process growing without bound. It is replaced once
+            // no tab is open.
+            if (browser && process.memoryUsage().rss > 1200 * 1024 * 1024)
+                recycle = true;
+            if (recycle && !running(browserKinds)) {
+                await browser?.close();
+                browser = undefined;
+                recycle = false;
+            }
+            if (input.onCheckpoint &&
+                input.checkpointEveryMs &&
+                Date.now() - lastCheckpoint > input.checkpointEveryMs) {
+                lastCheckpoint = Date.now();
+                await input.onCheckpoint(state.snapshot(phase));
+            }
         }
+        return false;
     };
     try {
-        emit({ type: "phase", phase: "crawl" });
-        state.enqueue("discover", "site");
-        await drain(crawlKinds);
-        emit({ type: "phase", phase: "lighthouse" });
-        const pages = state
-            .records(["page"])
-            .filter((p) => p.data?.http?.outcome === "ok" && p.data?.html !== false)
-            .sort((a, b) => a.key.localeCompare(b.key));
-        const { selected, templates } = selectLighthousePages(pages, options.browser && !scan.coverage.expired
-            ? options
-            : { lighthouse: "off", lighthousePerTemplate: 1 }, scan.rotation, scan.url);
-        scan.coverage.templates = templates;
-        scan.coverage.lighthouseEligible = pages.length;
-        scan.coverage.lighthouseSelected = selected;
-        scan.coverage.lighthouseSelection =
-            options.lighthouse === "all"
-                ? "Every eligible HTML page"
-                : options.lighthouse === "templates"
-                    ? `Homepage plus ${options.lighthousePerTemplate} rotating page per template (${templates} templates)`
-                    : "Lighthouse disabled";
-        for (const url of selected)
-            state.enqueue("lighthouse", url);
-        await drain(["lighthouse"]);
+        if (phase === "crawl") {
+            emit({ type: "phase", phase: "crawl" });
+            if (!input.resume)
+                state.enqueue("discover", "site");
+            if (await drain(crawlKinds))
+                return { paused: state.snapshot("crawl") };
+        }
+        if (phase !== "analysis")
+            emit({ type: "phase", phase: "lighthouse" });
+        if (phase === "crawl") {
+            const pages = state
+                .records(["page"])
+                .filter((p) => p.data?.http?.outcome === "ok" && p.data?.html !== false)
+                .sort((a, b) => a.key.localeCompare(b.key));
+            const { selected, templates } = selectLighthousePages(pages, options.browser && !scan.coverage.expired
+                ? options
+                : { lighthouse: "off", lighthousePerTemplate: 1 }, scan.rotation, scan.url);
+            scan.coverage.templates = templates;
+            scan.coverage.lighthouseEligible = pages.length;
+            scan.coverage.lighthouseSelected = selected;
+            scan.coverage.lighthouseSelection =
+                options.lighthouse === "all"
+                    ? "Every eligible HTML page"
+                    : options.lighthouse === "templates"
+                        ? `Homepage plus ${options.lighthousePerTemplate} rotating page per template (${templates} templates)`
+                        : "Lighthouse disabled";
+            for (const url of selected)
+                state.enqueue("lighthouse", url);
+            phase = "lighthouse";
+        }
+        if (phase === "lighthouse") {
+            if (await drain(["lighthouse"]))
+                return { paused: state.snapshot("lighthouse") };
+            phase = "analysis";
+        }
     }
     finally {
         await browser?.close();

package/dist/engine/state.d.ts

@@ -4,6 +4,17 @@ export interface StoredArtifact extends Artifact {
     sha256: string;
     bytes: number;
 }
+export interface ScanSnapshot {
+    version: 1;
+    phase: "crawl" | "lighthouse" | "analysis";
+    scan: Scan;
+    observations: Observation[];
+    urls: UrlRow[];
+    jobs: Job[];
+    artifacts: (Omit<StoredArtifact, "body"> & {
+        body: string;
+    })[];
+}
 export declare class ScanState {
     readonly scan: Scan;
     readonly observations: Map<string, Observation>;
@@ -12,6 +23,8 @@ export declare class ScanState {
     readonly jobs: Job[];
     private readonly jobIndex;
     constructor(scan: Scan);
+    snapshot(phase: ScanSnapshot["phase"]): ScanSnapshot;
+    static restore(snapshot: ScanSnapshot): ScanState;
     enqueue(kind: JobKind, name: string): void;
     next(kinds: JobKind[]): Job | undefined;
     pending(kinds: JobKind[]): number;

package/dist/engine/state.js

@@ -16,6 +16,40 @@ export class ScanState {
     constructor(scan) {
         this.scan = scan;
     }
+    snapshot(phase) {
+        return {
+            version: 1,
+            phase,
+            scan: this.scan,
+            observations: [...this.observations.values()],
+            urls: [...this.urls.values()],
+            jobs: this.jobs,
+            artifacts: [...this.artifacts.values()].map((a) => ({
+                ...a,
+                body: a.body.toString("base64"),
+            })),
+        };
+    }
+    static restore(snapshot) {
+        const state = new ScanState(snapshot.scan);
+        for (const o of snapshot.observations)
+            state.observations.set(key(o.kind, o.key), o);
+        for (const u of snapshot.urls)
+            state.urls.set(key(u.kind, u.url), u);
+        for (const job of snapshot.jobs) {
+            // A job that was running when the process stopped starts over.
+            if (job.status === "running")
+                job.status = "queued";
+            state.jobs.push(job);
+            state.jobIndex.set(key(job.kind, job.key), job);
+        }
+        for (const a of snapshot.artifacts)
+            state.artifacts.set(key(a.kind, a.key), {
+                ...a,
+                body: Buffer.from(a.body, "base64"),
+            });
+        return state;
+    }
     enqueue(kind, name) {
         const id = key(kind, name);
         if (this.jobIndex.has(id))

package/dist/engine.d.ts

@@ -1 +1,2 @@
-export { runScan, type RunInput, type RunOutput, type ScanEvent } from "./engine/run.js";
+export { runScan, runScanStep, type RunInput, type RunOutput, type ScanEvent, } from "./engine/run.js";
+export type { ScanSnapshot } from "./engine/state.js";

package/dist/engine.js

@@ -1 +1 @@
-export { runScan } from "./engine/run.js";
+export { runScan, runScanStep, } from "./engine/run.js";

package/package.json

@@ -1,10 +1,13 @@
 {
   "name": "@fulldotdev/scan",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Whole-site website scanner: crawls every page, renders it, runs Lighthouse and axe, and reports every problem it finds.",
   "license": "MIT",
   "type": "module",
-  "repository": "github:fulldotdev/scan",
+  "repository": {
+    "type": "git",
+    "url": "github:fulldotdev/scan"
+  },
   "homepage": "https://scan.full.dev",
   "bin": {
     "fullscan": "./bin/fullscan.js"
@@ -17,18 +20,14 @@
   },
   "types": "./dist/index.d.ts",
   "files": [
+    "SKILL.md",
     "bin",
-    "dist"
+    "dist",
+    "templates"
   ],
   "engines": {
     "node": ">=22"
   },
-  "scripts": {
-    "build": "tsc -p tsconfig.build.json",
-    "typecheck": "tsc --noEmit",
-    "test": "tsx --test tests/*.test.ts",
-    "scan": "tsx src/cli.ts"
-  },
   "dependencies": {
     "axe-core": "^4.13.0",
     "cheerio": "^1.2.0",
@@ -48,5 +47,11 @@
     "@types/node": "^24.13.4",
     "tsx": "^4.23.13",
     "typescript": "^7.0.2"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "typecheck": "tsc --noEmit",
+    "test": "tsx --test tests/*.test.ts",
+    "scan": "tsx src/cli.ts"
   }
-}
+}

package/templates/github-workflow.yml

@@ -0,0 +1,83 @@
+# Scan: copy to .github/workflows/scan.yml and add the repository secret
+# FULLSCAN_KEY (an API key from the account page on scan.full.dev).
+# Set SITE_URL to the live site.
+name: Scan
+
+on:
+  pull_request:
+  schedule:
+    - cron: "30 2 * * *" # daily crawl
+    - cron: "30 3 * * 0" # weekly full scan: browser pass, Lighthouse, external sources
+  workflow_dispatch:
+
+env:
+  SITE_URL: https://example.com
+
+permissions:
+  contents: read
+  pull-requests: write
+  statuses: read
+
+concurrency:
+  group: scan-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      # A pull request is scanned on its Netlify deploy preview, which Netlify
+      # reports as a commit status once the preview is ready.
+      - name: Find the URL to scan
+        id: target
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [ "${{ github.event_name }}" != "pull_request" ]; then
+            echo "url=$SITE_URL" >> "$GITHUB_OUTPUT"; exit 0
+          fi
+          for i in $(seq 1 60); do
+            url=$(gh api "repos/${{ github.repository }}/commits/${{ github.event.pull_request.head.sha }}/status" \
+              -q '.statuses[] | select(.context | test("^netlify/.+/deploy-preview$")) | select(.state == "success") | .target_url' | head -1)
+            [ -n "$url" ] && break
+            sleep 10
+          done
+          [ -n "$url" ] || { echo "No deploy preview found"; exit 1; }
+          echo "url=$url" >> "$GITHUB_OUTPUT"
+
+      - name: Scan
+        env:
+          FULLSCAN_KEY: ${{ secrets.FULLSCAN_KEY }}
+        run: |
+          mode=""
+          # Daily runs and pull requests crawl only; the weekly run is the full scan.
+          if [ "${{ github.event.schedule }}" = "30 2 * * *" ]; then mode="--quick"; fi
+          push=""
+          # Only scans of the live site go to the platform's history.
+          if [ "${{ github.event_name }}" != "pull_request" ] && [ -n "$FULLSCAN_KEY" ]; then push="--push"; fi
+          npx --yes @fulldotdev/scan "${{ steps.target.outputs.url }}" $mode $push --fail-on critical
+      - name: Summary
+        if: always()
+        run: cat scan/summary.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Comment on the pull request
+        if: always() && github.event_name == 'pull_request'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh pr comment ${{ github.event.pull_request.number }} -R ${{ github.repository }} --edit-last --body-file scan/summary.md \
+            || gh pr comment ${{ github.event.pull_request.number }} -R ${{ github.repository }} --body-file scan/summary.md
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: scan
+          path: |
+            scan/report.md
+            scan/result.json
+            scan/summary.md