@fulldotdev/scan 0.1.0

package/README.md

@@ -0,0 +1,79 @@
+# @fulldotdev/scan
+
+Whole-site website scanner. Give it a URL and it crawls every page, checks every linked file and destination, renders every page in headless Chrome, runs Lighthouse per page template and axe-core on every page, and writes every problem it finds as JSON and Markdown. It reports problems and their impact. It does not give fixes.
+
+```sh
+npx @fulldotdev/scan https://example.com
+```
+
+Requires Node 22 or newer. Chrome is installed with the package; set `CHROME_PATH` to use another one.
+
+## Usage
+
+```sh
+fullscan https://example.com                # whole site: crawl, browser pass, Lighthouse per template
+fullscan http://localhost:3000 --quick      # local dev server, crawl only
+fullscan https://example.com --lighthouse all
+fullscan https://example.com --pages 200 --budget 10
+fullscan https://example.com --previous scan/result.json   # change tracking against the last run
+fullscan https://example.com --fail-on critical             # exit 1 when a critical finding exists
+fullscan https://example.com --json > result.json
+fullscan https://example.com --push --key $FULLSCAN_KEY        # store the result on scan.full.dev
+```
+
+Options:
+
+| Flag | Meaning |
+| --- | --- |
+| `--out <dir>` | Output directory, default `./scan` |
+| `--quick` | Crawl only: no browser pass, no Lighthouse |
+| `--deep` | Deeper measurement: rendering observation and Lighthouse repeats |
+| `--lighthouse <mode>` | `templates` (default: homepage plus one rotating page per template), `all`, or `off` |
+| `--no-browser` | Skip the browser pass and Lighthouse |
+| `--no-external` | Skip external sources (RDAP, SSL Labs, Chrome UX Report, Safe Browsing, HSTS preload, certificate logs, blocklists) |
+| `--pages <n>` | Stop following new pages after n |
+| `--budget <minutes>` | Cancel remaining crawl work after this long; the report still builds |
+| `--previous <file>` | `result.json` of an earlier run, for new, resolved and unverified findings |
+| `--rotation <n>` | Template sample rotation; taken from `--previous` when absent |
+| `--fail-on critical\|warning` | Exit code 1 when such findings exist |
+| `--push` | Push the result to the platform after the scan and print the site and run id |
+| `--key <key>` | API key for `--push`; `FULLSCAN_KEY` is used when the flag is absent |
+| `--platform <url>` | Where `--push` sends the result, default `https://scan.full.dev` |
+| `--json` | Print the result JSON to stdout, no progress |
+| `--quiet` | No progress output |
+
+Local targets (localhost, private addresses, any port) are allowed automatically. External sources are then reported as not applicable.
+
+`--push` sends the scan, report, findings, pages and assets to your account on the platform, which creates the site on the first push and keeps the run in its history. Raw records and artifacts stay on your machine. Create a key on the account page of the platform.
+
+`GOOGLE_API_KEY` (a Google Cloud key with the Chrome UX Report API and Safe Browsing API enabled) enables real-user vitals and the Safe Browsing check.
+
+## Output
+
+- `result.json`: `scan` (options, coverage, status), `report` (overview, findings per topic, checks with outcome, coverage, changes), `findings` (every finding with all affected items), `pages` (one compact row per page: status, type, language, indexable, depth, word count, template, Lighthouse scores, finding counts), `assets` (every file and external destination with status, size and the pages that use it).
+- `report.md`: the same report as readable Markdown, made for people and language models.
+- `records.ndjson`: every raw observation (page extraction, rendered comparison, axe results, Lighthouse evidence, DNS, TLS, sitemaps and so on).
+- `artifacts/`: screenshots, retrieved sitemaps, robots.txt and agent files, with `index.json`.
+
+Every finding has a stable id, a topic, a severity (`critical`, `warning`, `hint`), a confidence (`confirmed` observed, `probable` inferred, `review` heuristic that needs a human), a problem sentence, an impact sentence and the affected pages with evidence. Every rule the scanner knows is listed under `checks` with its outcome (`passed`, `issue`, `review`, `not-applicable`, `unavailable`, `not-checked`), so absence of a finding is never mistaken for a pass of a check that did not run.
+
+## Topics
+
+Availability and security, crawling and indexability, content and structure, links and navigation, structured data, performance, accessibility and usability, business and profiles, AI and agent access.
+
+## Programmatic use
+
+```ts
+import { runScan, formatReport } from "@fulldotdev/scan"
+
+const { scan, report, findings, state } = await runScan({
+  url: "https://example.com",
+  options: { lighthouse: "templates" },
+  onEvent: (event) => console.error(event),
+})
+console.log(formatReport(report))
+```
+
+## Safety
+
+Requests resolve and pin public addresses and refuse private ranges, credentials and nonstandard ports unless the target itself is local. Robots is checked before every request. Every URL is fetched once per scan. Crawl-trap rules stop URLs over 2,000 characters, paths deeper than 12 segments and repeating segments. Forms are never submitted; the browser pass scrolls once and clicks one load-more control at most. A 429 slows the scan down for the rest of the run.

package/bin/fullscan.js

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import "../dist/cli.js";

package/dist/cli.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/cli.js

@@ -0,0 +1,239 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { parseArgs } from "node:util";
+import { runScan } from "./engine/run.js";
+import { templateKey } from "./engine/select.js";
+import { version } from "./engine/types.js";
+import { formatReport } from "./report/build.js";
+import { assetInventory, pageRows } from "./report/evidence.js";
+import { publicReport } from "./report/rules.js";
+const usage = `fullscan ${version}
+
+Usage: fullscan <url> [options]
+
+Scans a whole website: every page is crawled and rendered, Lighthouse runs
+per template, and every problem found is written as JSON and Markdown.
+
+Options:
+  --out <dir>            Output directory (default: ./scan)
+  --quick                Crawl only: no browser pass, no Lighthouse
+  --deep                 Deeper measurement: rendering observation, Lighthouse repeats
+  --lighthouse <mode>    templates (default), all, or off
+  --no-browser           Skip the browser pass and Lighthouse
+  --no-external          Skip external sources (RDAP, SSL Labs, CrUX, ...)
+  --pages <n>            Stop following new pages after n
+  --budget <minutes>     Cancel remaining crawl work after this long
+  --previous <file>      result.json of an earlier run, for change tracking
+  --rotation <n>         Template sample rotation (default: from --previous, else 0)
+  --fail-on <severity>   Exit 1 when findings of this severity exist: critical or warning
+  --push                 Push the result to the platform after the scan
+  --key <key>            API key for --push (default: FULLSCAN_KEY)
+  --platform <url>       Platform to push to (default: https://scan.full.dev)
+  --json                 Print the result JSON to stdout instead of progress
+  --quiet                No progress output
+  -h, --help             Show this help
+
+Local targets (localhost, private addresses, any port) are allowed
+automatically; external sources are then reported as not applicable.
+`;
+const { values, positionals } = parseArgs({
+    allowPositionals: true,
+    options: {
+        out: { type: "string", default: "scan" },
+        quick: { type: "boolean", default: false },
+        deep: { type: "boolean", default: false },
+        lighthouse: { type: "string" },
+        browser: { type: "boolean", default: true },
+        external: { type: "boolean", default: true },
+        pages: { type: "string" },
+        budget: { type: "string" },
+        previous: { type: "string" },
+        rotation: { type: "string" },
+        "fail-on": { type: "string" },
+        push: { type: "boolean", default: false },
+        key: { type: "string" },
+        platform: { type: "string", default: "https://scan.full.dev" },
+        json: { type: "boolean", default: false },
+        quiet: { type: "boolean", default: false },
+        help: { type: "boolean", short: "h", default: false },
+    },
+    allowNegative: true,
+});
+const target = positionals[0];
+if (values.help || !target) {
+    process.stdout.write(usage);
+    process.exit(values.help ? 0 : 1);
+}
+const url = /^https?:\/\//i.test(target) ? target : `https://${target}`;
+const host = new URL(url).hostname;
+const local = host === "localhost" ||
+    host.endsWith(".localhost") ||
+    /^(127\.|10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.|0\.0\.0\.0|\[::1\])/.test(host);
+const options = {
+    deepScan: values.deep,
+    browser: values.browser && !values.quick,
+    lighthouse: values.quick
+        ? "off"
+        : (values.lighthouse ??
+            "templates"),
+    external: values.external,
+    allowLocal: local,
+    ...(values.pages ? { maxPages: Number(values.pages) } : {}),
+};
+let previous;
+if (values.previous) {
+    const file = JSON.parse(await readFile(values.previous, "utf8"));
+    previous = {
+        id: file.scan?.id ?? "previous",
+        createdAt: file.scan?.createdAt ?? file.report?.createdAt,
+        report: file.report,
+        findings: file.findings ?? [],
+        pages: (file.pages ?? []).map((p) => ({
+            key: p.url,
+            title: p.title,
+            outcome: p.outcome,
+            status: p.status,
+            words: p.wordCount,
+            canonical: p.canonical,
+            language: p.language,
+        })),
+        lastScores: file.report?.scores?.carried ? null : file.report?.scores,
+    };
+}
+const rotation = values.rotation
+    ? Number(values.rotation)
+    : previous
+        ? Number(JSON.parse(await readFile(values.previous, "utf8")).scan?.rotation ??
+            -1) + 1
+        : 0;
+const quiet = values.quiet || values.json;
+const log = (line) => {
+    if (!quiet)
+        process.stderr.write(`${line}\n`);
+};
+const onEvent = (event) => {
+    switch (event.type) {
+        case "start":
+            log(`Scanning ${event.scan.url}`);
+            break;
+        case "phase":
+            log(`Phase: ${event.phase}`);
+            break;
+        case "job":
+            if (event.outcome === "completed")
+                log(`  ${event.kind} ${event.key} (${event.durationMs} ms, ${event.pending} left)`);
+            else
+                log(`  ${event.kind} ${event.key} ${event.outcome}: ${event.error} (attempt ${event.attempt})`);
+            break;
+    }
+};
+const started = Date.now();
+const output = await runScan({
+    url,
+    options,
+    rotation,
+    previous,
+    budgetMs: values.budget ? Number(values.budget) * 60000 : undefined,
+    onEvent,
+});
+const { scan, report, findings, state } = output;
+const records = state.records();
+const urls = state.urlRows();
+const pages = pageRows(records, urls, findings, templateKey);
+const assets = assetInventory(records, urls, scan.url);
+const result = {
+    version: 1,
+    tool: { name: "@fulldotdev/scan", version },
+    scan,
+    report: publicReport(report),
+    findings,
+    pages,
+    assets,
+};
+const out = values.out;
+await mkdir(join(out, "artifacts"), { recursive: true });
+await writeFile(join(out, "result.json"), JSON.stringify(result, null, 2));
+await writeFile(join(out, "report.md"), formatReport(report) + "\n");
+await writeFile(join(out, "records.ndjson"), records.map((r) => JSON.stringify(r)).join("\n") + "\n");
+const index = {};
+for (const artifact of state.artifacts.values()) {
+    const name = `${artifact.sha256.slice(0, 16)}${extension(artifact.contentType)}`;
+    await writeFile(join(out, "artifacts", name), artifact.body);
+    index[name] = {
+        kind: artifact.kind,
+        key: artifact.key,
+        contentType: artifact.contentType,
+    };
+}
+await writeFile(join(out, "artifacts", "index.json"), JSON.stringify(index, null, 2));
+// The push carries the result only: records and artifacts stay local.
+let pushed;
+if (values.push) {
+    const key = values.key ?? process.env.FULLSCAN_KEY;
+    if (!key) {
+        process.stderr.write("A key is required to push: pass --key or set FULLSCAN_KEY.\n");
+        process.exit(1);
+    }
+    const platform = values.platform.replace(/\/+$/, "");
+    const response = await fetch(`${platform}/api/command`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${key}`,
+        },
+        body: JSON.stringify({
+            operation: "runs.push",
+            args: {
+                url: scan.url,
+                scan,
+                report: result.report,
+                findings,
+                pages,
+                assets,
+            },
+        }),
+    });
+    const payload = (await response.json().catch(() => null));
+    if (!response.ok || !payload?.result) {
+        process.stderr.write(`Push failed: ${payload?.error?.message ?? `${response.status} ${response.statusText}`}\n`);
+        process.exit(1);
+    }
+    pushed = payload.result;
+    if (values.json)
+        process.stderr.write(`Pushed to ${platform}: site ${pushed.siteId}, run ${pushed.runId}\n`);
+}
+if (values.json)
+    process.stdout.write(JSON.stringify(result) + "\n");
+else {
+    const c = report.counts;
+    process.stdout.write([
+        `${scan.url}: ${report.pages.observed} pages, ${c.critical} critical, ${c.warning} warnings, ${c.hint ?? 0} hints (${scan.status}, ${Math.round((Date.now() - started) / 1000)} s)`,
+        `Report: ${join(out, "report.md")}`,
+        `Result: ${join(out, "result.json")}`,
+        ...(pushed
+            ? [`Pushed: site ${pushed.siteId}, run ${pushed.runId}`]
+            : []),
+    ].join("\n") + "\n");
+}
+if (values["fail-on"]) {
+    const c = report.counts;
+    const failed = values["fail-on"] === "critical"
+        ? c.critical > 0
+        : c.critical > 0 || c.warning > 0;
+    process.exit(failed ? 1 : 0);
+}
+function extension(contentType) {
+    if (/png/.test(contentType))
+        return ".png";
+    if (/jpe?g/.test(contentType))
+        return ".jpg";
+    if (/webp/.test(contentType))
+        return ".webp";
+    if (/xml/.test(contentType))
+        return ".xml";
+    if (/json/.test(contentType))
+        return ".json";
+    if (/markdown/.test(contentType))
+        return ".md";
+    return ".txt";
+}

package/dist/engine/analysis.d.ts

@@ -0,0 +1,2 @@
+import type { JobResult, Observation, Scan, UrlRow } from "./types.js";
+export declare function analyze(scan: Scan, records: Observation[], urls: UrlRow[]): Promise<JobResult>;