@fulldotdev/scan 0.2.0 → 0.3.0

package/dist/cli.js

@@ -29,6 +29,8 @@ Options:
   --push                 Push the result to the platform after the scan
   --key <key>            API key for --push (default: FULLSCAN_KEY)
   --platform <url>       Platform to push to (default: https://scan.full.dev)
+  --run <id>             Fill this queued run of the platform with the result
+  --scheduled            Store the result as a run of the nightly schedule
   --json                 Print the result JSON to stdout instead of progress
   --quiet                No progress output
   -h, --help             Show this help
@@ -53,6 +55,8 @@ const { values, positionals } = parseArgs({
         push: { type: "boolean", default: false },
         key: { type: "string" },
         platform: { type: "string", default: "https://scan.full.dev" },
+        run: { type: "string" },
+        scheduled: { type: "boolean", default: false },
         json: { type: "boolean", default: false },
         quiet: { type: "boolean", default: false },
         help: { type: "boolean", short: "h", default: false },
@@ -80,7 +84,41 @@ const options = {
     allowLocal: local,
     ...(values.pages ? { maxPages: Number(values.pages) } : {}),
 };
+const key = values.key ?? process.env.FULLSCAN_KEY;
+const platform = values.platform.replace(/\/+$/, "");
+// One call to the platform's typed API. Returns the result or throws.
+async function callPlatform(path, operation, args) {
+    const response = await fetch(`${platform}/api/${path}`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${key}`,
+        },
+        body: JSON.stringify({ operation, args }),
+    });
+    const payload = (await response.json().catch(() => null));
+    if (!response.ok)
+        throw new Error(payload?.error?.message ?? `${response.status} ${response.statusText}`);
+    return payload?.result;
+}
 let previous;
+let previousRotation;
+// With a key and no file, the platform tells the scan what the site looked
+// like last time, so changes are tracked on a runner just as well.
+if (!values.previous && key) {
+    try {
+        const result = await callPlatform("query", "runs.previous", { url });
+        if (result) {
+            const { rotation: stored, ...rest } = result;
+            previous = rest;
+            if (typeof stored === "number")
+                previousRotation = stored;
+        }
+    }
+    catch (error) {
+        process.stderr.write(`Could not read the previous scan: ${error instanceof Error ? error.message : String(error)}\n`);
+    }
+}
 if (values.previous) {
     const file = JSON.parse(await readFile(values.previous, "utf8"));
     previous = {
@@ -102,10 +140,12 @@ if (values.previous) {
 }
 const rotation = values.rotation
     ? Number(values.rotation)
-    : previous
-        ? Number(JSON.parse(await readFile(values.previous, "utf8")).scan?.rotation ??
-            -1) + 1
-        : 0;
+    : previousRotation !== undefined
+        ? previousRotation
+        : values.previous
+            ? Number(JSON.parse(await readFile(values.previous, "utf8")).scan?.rotation ??
+                -1) + 1
+            : 0;
 const quiet = values.quiet || values.json;
 const log = (line) => {
     if (!quiet)
@@ -167,41 +207,100 @@ for (const artifact of state.artifacts.values()) {
     };
 }
 await writeFile(join(out, "artifacts", "index.json"), JSON.stringify(index, null, 2));
-// The push carries the result only: records and artifacts stay local.
+// The push carries the result; the observations and the screenshots follow
+// it in their own calls, so the run on the platform is as complete as one
+// the platform ran itself.
 let pushed;
 if (values.push) {
-    const key = values.key ?? process.env.FULLSCAN_KEY;
     if (!key) {
         process.stderr.write("A key is required to push: pass --key or set FULLSCAN_KEY.\n");
         process.exit(1);
     }
-    const platform = values.platform.replace(/\/+$/, "");
-    const response = await fetch(`${platform}/api/command`, {
-        method: "POST",
-        headers: {
-            "Content-Type": "application/json",
-            Authorization: `Bearer ${key}`,
-        },
-        body: JSON.stringify({
-            operation: "runs.push",
-            args: {
-                url: scan.url,
-                scan,
-                report: result.report,
-                findings,
-                pages,
-                assets,
-            },
-        }),
-    });
-    const payload = (await response.json().catch(() => null));
-    if (!response.ok || !payload?.result) {
-        process.stderr.write(`Push failed: ${payload?.error?.message ?? `${response.status} ${response.statusText}`}\n`);
+    try {
+        pushed = await callPlatform("command", "runs.push", {
+            url: scan.url,
+            scan,
+            report: result.report,
+            findings,
+            pages,
+            assets,
+            ...(values.run ? { runId: values.run } : {}),
+            ...(values.scheduled ? { scheduled: true } : {}),
+        });
+    }
+    catch (error) {
+        process.stderr.write(`Push failed: ${error instanceof Error ? error.message : String(error)}\n`);
+        process.exit(1);
+    }
+    if (!pushed) {
+        process.stderr.write("Push failed: the platform stored no run.\n");
         process.exit(1);
     }
-    pushed = payload.result;
     if (values.json)
         process.stderr.write(`Pushed to ${platform}: site ${pushed.siteId}, run ${pushed.runId}\n`);
+    await pushRecords(pushed.runId);
+    await pushScreenshots(pushed.runId);
+}
+// Observations go up in batches that stay under 3 MB of JSON, because the
+// platform's request limit is a few megabytes. A batch that fails costs the
+// page detail of this run, never the run itself.
+async function pushRecords(runId) {
+    const limit = 3 * 1024 * 1024;
+    const batches = [[]];
+    let size = 0;
+    for (const record of records) {
+        const bytes = Buffer.byteLength(JSON.stringify(record));
+        if (size + bytes > limit && batches[batches.length - 1].length) {
+            batches.push([]);
+            size = 0;
+        }
+        batches[batches.length - 1].push(record);
+        size += bytes;
+    }
+    for (const [index, batch] of batches.entries()) {
+        const final = index === batches.length - 1;
+        try {
+            await callPlatform("command", "runs.pushRecords", {
+                runId,
+                records: batch,
+                final,
+            });
+        }
+        catch (error) {
+            process.stderr.write(`Could not push ${batch.length} observations: ${error instanceof Error ? error.message : String(error)}\n`);
+        }
+    }
+}
+// Screenshots are raw bytes, so they go to their own endpoint instead of
+// swelling the JSON push by a third.
+async function pushScreenshots(runId) {
+    for (const artifact of state.artifacts.values()) {
+        if (artifact.kind !== "screenshot")
+            continue;
+        try {
+            const query = new URLSearchParams({
+                runId,
+                kind: "screenshot",
+                key: artifact.key,
+            });
+            const response = await fetch(`${platform}/api/artifacts/upload?${query.toString()}`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": artifact.contentType,
+                    Authorization: `Bearer ${key}`,
+                },
+                body: new Uint8Array(artifact.body),
+            });
+            if (!response.ok) {
+                const payload = (await response.json().catch(() => null));
+                throw new Error(payload?.error?.message ??
+                    `${response.status} ${response.statusText}`);
+            }
+        }
+        catch (error) {
+            process.stderr.write(`Could not upload the screenshot of ${artifact.key}: ${error instanceof Error ? error.message : String(error)}\n`);
+        }
+    }
 }
 if (values.json)
     process.stdout.write(JSON.stringify(result) + "\n");

package/dist/engine/browser.js

@@ -82,8 +82,7 @@ const axeTags = [
 export async function collectBrowser(scan, url, http, session) {
     await resolvePublic(new URL(url).hostname);
     const robots = await http.getRobots(url);
-    if (!robots.allowed ||
-        robots.parser?.isAllowed(url, "FulldevScan") === false)
+    if (!robots.allowed || robots.parser?.isAllowed(url, "FulldevScan") === false)
         return {
             observations: [
                 {
@@ -95,6 +94,9 @@ export async function collectBrowser(scan, url, http, session) {
         };
     const { proxy, browser, chromeVersion } = session;
     const blockedBefore = proxy.blocked.length;
+    // Hosts this page asked for, so blocked requests of a tab rendering at
+    // the same time are not counted here.
+    const requestedHosts = new Set();
     const result = { observations: [], artifacts: [], candidates: [] };
     const errors = [];
     let page;
@@ -104,8 +106,9 @@ export async function collectBrowser(scan, url, http, session) {
             stage: "budget",
             error: `Browser job exceeded ${budgetSeconds} seconds`,
         });
-        // Closing the browser aborts the page; the worker reopens a session.
-        void browser.close();
+        // Closing the tab aborts this page and leaves its neighbours alone; a
+        // tab that will not close takes the browser with it.
+        void (page ? page.close().catch(() => browser.close()) : browser.close());
     }, budgetSeconds * 1000);
     watchdog.unref();
     try {
@@ -136,6 +139,12 @@ export async function collectBrowser(scan, url, http, session) {
         const cdp = await page.createCDPSession();
         await cdp.send("Network.enable");
         cdp.on("Network.requestWillBeSent", (e) => {
+            try {
+                requestedHosts.add(new URL(e.request?.url ?? "").hostname);
+            }
+            catch {
+                // Not a URL with a host (data:, blob:).
+            }
             if (staticTypes.test(e.type) && /^https?:/.test(e.request?.url ?? ""))
                 staticResources.add(e.request.url);
         });
@@ -624,7 +633,9 @@ export async function collectBrowser(scan, url, http, session) {
         data: {
             status: errors.length ? "partial" : "completed",
             errors,
-            blockedNetwork: proxy.blocked.slice(blockedBefore),
+            blockedNetwork: proxy.blocked
+                .slice(blockedBefore)
+                .filter((entry) => [...requestedHosts].some((host) => entry.target.includes(host))),
         },
     });
     return result;

package/dist/engine/proxy.js

@@ -7,6 +7,8 @@ export async function createEgressProxy() {
     const sockets = new Set();
     const remember = (socket) => {
         sockets.add(socket);
+        // Several checks share a kept-alive socket; each adds its own listeners.
+        socket.setMaxListeners(50);
         socket.on("close", () => sockets.delete(socket));
         socket.setTimeout(30000, () => socket.destroy());
     };

package/dist/engine/run.js

@@ -49,13 +49,23 @@ export async function runScanStep(input) {
     const http = new HttpClient(options, undefined, undefined, session);
     const deadline = input.budgetMs ? Date.now() + input.budgetMs : Infinity;
     let browser;
+    let opening;
+    // One Chrome for the scan. Jobs that ask at the same moment share the
+    // launch instead of starting two browsers.
     const browserSession = async () => {
-        const { openBrowserSession } = await import("./browser.js");
-        if (browser && !browser.connected) {
-            await browser.close();
-            browser = undefined;
-        }
-        return (browser ??= await openBrowserSession());
+        if (browser?.connected)
+            return browser;
+        opening ??= (async () => {
+            const { openBrowserSession } = await import("./browser.js");
+            await browser?.close();
+            browser = await openBrowserSession();
+            opening = undefined;
+            return browser;
+        })().catch((error) => {
+            opening = undefined;
+            throw error;
+        });
+        return opening;
     };
     const run = async (job) => {
         switch (job.kind) {
@@ -147,28 +157,53 @@ export async function runScanStep(input) {
                 pending: state.pending(kinds),
             });
         }
-        // Chromium and Lighthouse leave memory behind; a fresh browser for the
-        // next page is cheaper than the process growing without bound.
-        if (["browser", "lighthouse"].includes(job.kind) &&
-            browser &&
-            process.memoryUsage().rss > 1200 * 1024 * 1024) {
-            await browser.close();
-            browser = undefined;
-        }
     };
+    // How many jobs of a kind run at once. Link and file checks mostly wait on
+    // the network; two pages render side by side as tabs of one Chrome. Pages
+    // of the scanned site are fetched one at a time so the site sets the pace,
+    // and Lighthouse runs alone because a busy CPU worsens its scores.
+    const concurrency = {
+        resource: 6,
+        browser: 2,
+    };
+    const browserKinds = ["browser", "lighthouse"];
     let lastCheckpoint = Date.now();
     // True when the budget ran out and the scan should pause here.
     const drain = async (kinds) => {
-        let job;
-        while ((job = state.next(kinds))) {
-            if (Date.now() > deadline || input.signal?.aborted) {
+        const inFlight = new Map();
+        const running = (list) => [...inFlight.keys()].filter((j) => list.includes(j.kind)).length;
+        let recycle = false;
+        for (;;) {
+            const stopping = Date.now() > deadline || !!input.signal?.aborted;
+            let job;
+            while (!stopping &&
+                (job = state.next(kinds.filter((kind) => running([kind]) < (concurrency[kind] ?? 1) &&
+                    // No new tab while Chrome waits to be replaced.
+                    !(recycle && browserKinds.includes(kind)))))) {
+                const current = job;
+                current.status = "running";
+                inFlight.set(current, execute(current, kinds).finally(() => inFlight.delete(current)));
+            }
+            if (!inFlight.size) {
+                if (!stopping || !state.pending(kinds))
+                    break;
                 if (input.pauseAtBudget)
                     return true;
                 state.cancelQueued();
                 scan.coverage.expired = true;
                 break;
             }
-            await execute(job, kinds);
+            await Promise.race(inFlight.values());
+            // Chromium and Lighthouse leave memory behind; a fresh browser is
+            // cheaper than the process growing without bound. It is replaced once
+            // no tab is open.
+            if (browser && process.memoryUsage().rss > 1200 * 1024 * 1024)
+                recycle = true;
+            if (recycle && !running(browserKinds)) {
+                await browser?.close();
+                browser = undefined;
+                recycle = false;
+            }
             if (input.onCheckpoint &&
                 input.checkpointEveryMs &&
                 Date.now() - lastCheckpoint > input.checkpointEveryMs) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fulldotdev/scan",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Whole-site website scanner: crawls every page, renders it, runs Lighthouse and axe, and reports every problem it finds.",
   "license": "MIT",
   "type": "module",