@eightmedia/flightcheck 0.0.0-stage → 0.1.0

package/dist/crawler/defaults.d.ts

@@ -0,0 +1,11 @@
+export declare const DEFAULT_MAX_PAGES = 1000;
+export declare const DEFAULT_DELAY_MS = 500;
+export declare const DEFAULT_TIMEOUT_MS = 10000;
+export declare const DEFAULT_CONCURRENCY = 1;
+export declare const DEFAULT_BURST_DELAY_MS = 0;
+export declare const DEFAULT_OUTPUT_DIR = "./flightcheck";
+export declare const DEFAULT_SITEMAP_URL_SUFFIX = "/sitemap.xml";
+export declare const DEFAULT_MAX_SITEMAPS = 20;
+export declare const DEFAULT_MAX_SITEMAP_URLS = 10000;
+export declare const DEFAULT_USER_AGENT = "h1-crawler/2.0 (+https://local-script)";
+//# sourceMappingURL=defaults.d.ts.map

package/dist/crawler/defaults.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"defaults.d.ts","sourceRoot":"","sources":["../../src/crawler/defaults.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,iBAAiB,OAAO,CAAC;AACtC,eAAO,MAAM,gBAAgB,MAAM,CAAC;AACpC,eAAO,MAAM,kBAAkB,QAAQ,CAAC;AACxC,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,eAAO,MAAM,sBAAsB,IAAI,CAAC;AACxC,eAAO,MAAM,kBAAkB,kBAAkB,CAAC;AAClD,eAAO,MAAM,0BAA0B,iBAAiB,CAAC;AACzD,eAAO,MAAM,oBAAoB,KAAK,CAAC;AACvC,eAAO,MAAM,wBAAwB,QAAQ,CAAC;AAC9C,eAAO,MAAM,kBAAkB,2CAA2C,CAAC"}

package/dist/crawler/defaults.js

@@ -0,0 +1,10 @@
+export const DEFAULT_MAX_PAGES = 1000;
+export const DEFAULT_DELAY_MS = 500;
+export const DEFAULT_TIMEOUT_MS = 10000;
+export const DEFAULT_CONCURRENCY = 1;
+export const DEFAULT_BURST_DELAY_MS = 0;
+export const DEFAULT_OUTPUT_DIR = "./flightcheck";
+export const DEFAULT_SITEMAP_URL_SUFFIX = "/sitemap.xml";
+export const DEFAULT_MAX_SITEMAPS = 20;
+export const DEFAULT_MAX_SITEMAP_URLS = 10000;
+export const DEFAULT_USER_AGENT = "h1-crawler/2.0 (+https://local-script)";

package/dist/crawler/extractors.d.ts

@@ -0,0 +1,5 @@
+import { type VerificationRecord } from "./schema.js";
+import { type CheckGroup } from "./checkGroups.js";
+export declare function extractVerificationRecord(html: string, url: string, status: number | null, contentType: string, activeGroups?: Set<CheckGroup>): VerificationRecord;
+export declare function extractInternalLinks(html: string, currentUrl: string, origin: string): Set<string>;
+//# sourceMappingURL=extractors.d.ts.map

package/dist/crawler/extractors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractors.d.ts","sourceRoot":"","sources":["../../src/crawler/extractors.ts"],"names":[],"mappings":"AACA,OAAO,EAAiC,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAGrF,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAwBjE,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,WAAW,EAAE,MAAM,EACnB,YAAY,GAAE,GAAG,CAAC,UAAU,CAAyB,GACpD,kBAAkB,CAgFpB;AAmDD,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CA8BlG"}

package/dist/crawler/extractors.js

@@ -0,0 +1,169 @@
+import * as cheerio from "cheerio";
+import { createEmptyVerificationRecord } from "./schema.js";
+import { extractTracking } from "./tracking.js";
+import { resolveUrlMaybe } from "./urlUtils.js";
+import { CHECK_GROUPS } from "./checkGroups.js";
+import { extractA11yStaticIssues, extractImageIssues } from "./tier1Extractors.js";
+function textOrEmpty(value) {
+    return (value ?? "").trim();
+}
+function attrOrEmpty($, selector, attrName, options) {
+    if (options?.containsRel) {
+        const containsRel = options.containsRel.toLowerCase();
+        const node = $(selector)
+            .toArray()
+            .map((el) => $(el))
+            .find((nodeEl) => textOrEmpty(nodeEl.attr("rel")).toLowerCase().includes(containsRel));
+        return textOrEmpty(node?.attr(attrName));
+    }
+    return textOrEmpty($(selector).first().attr(attrName));
+}
+export function extractVerificationRecord(html, url, status, contentType, activeGroups = new Set(CHECK_GROUPS)) {
+    const empty = createEmptyVerificationRecord(url, status);
+    if (!contentType.includes("text/html")) {
+        return {
+            ...empty,
+            note: `Skipped non-HTML content-type: ${contentType || "unknown"}`,
+        };
+    }
+    const $ = cheerio.load(html);
+    const result = { ...empty };
+    if (activeGroups.has("meta")) {
+        const title = textOrEmpty($("title").first().text());
+        result.title = title;
+        result.h1 = textOrEmpty($("h1").first().text());
+        result.metaDescription = attrOrEmpty($, 'meta[name="description"]', "content");
+        result.robots = attrOrEmpty($, 'meta[name="robots"]', "content");
+        result.template = attrOrEmpty($, 'meta[name="template"]', "content");
+        result.colorScheme = attrOrEmpty($, 'meta[name="color-scheme"]', "content");
+        result.themeColor = attrOrEmpty($, 'meta[name="theme-color"]', "content");
+        result.viewport = attrOrEmpty($, 'meta[name="viewport"]', "content");
+        result.mainCount = $("main").length;
+        result.language = textOrEmpty($("html").first().attr("lang"));
+        result.charset = textOrEmpty($("meta[charset]").first().attr("charset"));
+        const doctypeMatch = html.match(/<!doctype\s+([^>]+)>/i);
+        result.doctype = textOrEmpty(doctypeMatch?.[1]);
+        result.canonical = resolveUrlMaybe(attrOrEmpty($, 'link[rel="canonical"]', "href"), url);
+    }
+    if (activeGroups.has("social")) {
+        result.og = {
+            title: attrOrEmpty($, 'meta[property="og:title"]', "content"),
+            description: attrOrEmpty($, 'meta[property="og:description"]', "content"),
+            image: resolveUrlMaybe(attrOrEmpty($, 'meta[property="og:image"]', "content"), url),
+            url: resolveUrlMaybe(attrOrEmpty($, 'meta[property="og:url"]', "content"), url),
+            type: attrOrEmpty($, 'meta[property="og:type"]', "content"),
+            siteName: attrOrEmpty($, 'meta[property="og:site_name"]', "content"),
+        };
+        result.twitter = {
+            card: attrOrEmpty($, 'meta[name="twitter:card"]', "content"),
+            title: attrOrEmpty($, 'meta[name="twitter:title"]', "content"),
+            description: attrOrEmpty($, 'meta[name="twitter:description"]', "content"),
+            image: resolveUrlMaybe(attrOrEmpty($, 'meta[name="twitter:image"]', "content"), url),
+        };
+    }
+    if (activeGroups.has("meta")) {
+        result.metaTitle = textOrEmpty(result.og.title || result.twitter.title || result.title);
+    }
+    if (activeGroups.has("icons")) {
+        result.favicon = resolveUrlMaybe(attrOrEmpty($, "link[rel]", "href", { containsRel: "icon" }), url);
+        result.icons = $("link[rel]")
+            .toArray()
+            .map((el) => $(el))
+            .filter((el) => textOrEmpty(el.attr("rel")).toLowerCase().includes("icon"))
+            .map((el) => ({
+            rel: textOrEmpty(el.attr("rel")).toLowerCase(),
+            href: resolveUrlMaybe(textOrEmpty(el.attr("href")), url),
+            type: textOrEmpty(el.attr("type")),
+            sizes: textOrEmpty(el.attr("sizes")),
+            media: textOrEmpty(el.attr("media")),
+        }));
+    }
+    if (activeGroups.has("tracking")) {
+        result.tracking = extractTracking(html);
+    }
+    if (activeGroups.has("images")) {
+        result.images = { issues: extractImageIssues($, url) };
+    }
+    if (activeGroups.has("a11y-static")) {
+        result.a11yStatic = { issues: extractA11yStaticIssues($) };
+    }
+    return result;
+}
+const NON_HTML_EXTENSIONS = new Set([
+    "pdf",
+    "zip",
+    "rar",
+    "gz",
+    "tar",
+    "7z",
+    "doc",
+    "docx",
+    "xls",
+    "xlsx",
+    "ppt",
+    "pptx",
+    "csv",
+    "rtf",
+    "png",
+    "jpg",
+    "jpeg",
+    "gif",
+    "webp",
+    "svg",
+    "ico",
+    "bmp",
+    "tiff",
+    "mp3",
+    "wav",
+    "ogg",
+    "mp4",
+    "webm",
+    "mov",
+    "avi",
+    "mkv",
+    "woff",
+    "woff2",
+    "ttf",
+    "eot",
+    "dmg",
+    "exe",
+    "apk",
+]);
+function hasNonHtmlExtension(pathname) {
+    const lastSegment = pathname.split("/").pop() ?? "";
+    const dotIndex = lastSegment.lastIndexOf(".");
+    if (dotIndex === -1)
+        return false;
+    const extension = lastSegment.slice(dotIndex + 1).toLowerCase();
+    return NON_HTML_EXTENSIONS.has(extension);
+}
+export function extractInternalLinks(html, currentUrl, origin) {
+    const $ = cheerio.load(html);
+    const links = new Set();
+    $("a[href]").each((_, element) => {
+        const rawHref = textOrEmpty($(element).attr("href"));
+        if (!rawHref ||
+            rawHref.startsWith("#") ||
+            rawHref.startsWith("mailto:") ||
+            rawHref.startsWith("tel:") ||
+            rawHref.startsWith("javascript:")) {
+            return;
+        }
+        try {
+            const absolute = new URL(rawHref, currentUrl);
+            if (absolute.origin !== origin)
+                return;
+            if (absolute.protocol !== "http:" && absolute.protocol !== "https:")
+                return;
+            if (hasNonHtmlExtension(absolute.pathname))
+                return;
+            absolute.hash = "";
+            absolute.search = "";
+            links.add(absolute.toString());
+        }
+        catch {
+            // Ignore malformed links.
+        }
+    });
+    return links;
+}

package/dist/crawler/jsAssets.d.ts

@@ -0,0 +1,2 @@
+export declare function extractSameOriginScripts(html: string, baseUrl: string, origin: string): string[];
+//# sourceMappingURL=jsAssets.d.ts.map

package/dist/crawler/jsAssets.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsAssets.d.ts","sourceRoot":"","sources":["../../src/crawler/jsAssets.ts"],"names":[],"mappings":"AAEA,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAkBhG"}

package/dist/crawler/jsAssets.js

@@ -0,0 +1,22 @@
+import * as cheerio from "cheerio";
+export function extractSameOriginScripts(html, baseUrl, origin) {
+    const $ = cheerio.load(html);
+    const urls = new Set();
+    $("script[src]").each((_, element) => {
+        const src = ($(element).attr("src") ?? "").trim();
+        if (!src)
+            return;
+        try {
+            const absolute = new URL(src, baseUrl);
+            if (absolute.origin !== origin)
+                return;
+            if (absolute.protocol !== "http:" && absolute.protocol !== "https:")
+                return;
+            urls.add(absolute.toString());
+        }
+        catch {
+            // Ignore malformed src values.
+        }
+    });
+    return [...urls];
+}

package/dist/crawler/outputPaths.d.ts

@@ -0,0 +1,14 @@
+export declare function formatRunTimestamp(date: Date): string;
+export declare function hostnameFromUrl(url: string): string;
+export declare function buildDefaultOutputPaths(startUrl: string, now?: Date): {
+    rawJson: string;
+    reportMd: string;
+    reportJson: string;
+};
+export declare function defaultReportPathForInput(inputPath: string): string;
+export declare function defaultReportJsonPathForInput(inputPath: string): string;
+/** @deprecated Fix-manifest output is removed; kept for callers that still resolve paths. */
+export declare function defaultManifestPathForInput(inputPath: string): string;
+/** @deprecated Agent-jobs output is removed; kept for callers that still resolve paths. */
+export declare function defaultAgentJobsPathForInput(inputPath: string): string;
+//# sourceMappingURL=outputPaths.d.ts.map

package/dist/crawler/outputPaths.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"outputPaths.d.ts","sourceRoot":"","sources":["../../src/crawler/outputPaths.ts"],"names":[],"mappings":"AAEA,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGrD;AAED,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAGnD;AAED,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,MAAM,EAChB,GAAG,OAAa,GACf;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAS3D;AAqBD,wBAAgB,yBAAyB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEnE;AAED,wBAAgB,6BAA6B,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEvE;AAED,6FAA6F;AAC7F,wBAAgB,2BAA2B,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAErE;AAED,2FAA2F;AAC3F,wBAAgB,4BAA4B,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEtE"}

package/dist/crawler/outputPaths.js

@@ -0,0 +1,51 @@
+import { DEFAULT_OUTPUT_DIR } from "./defaults.js";
+export function formatRunTimestamp(date) {
+    const pad = (value) => String(value).padStart(2, "0");
+    return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}.${pad(date.getHours())}${pad(date.getMinutes())}`;
+}
+export function hostnameFromUrl(url) {
+    const hostname = new URL(url).hostname;
+    return hostname.startsWith("www.") ? hostname.slice(4) : hostname;
+}
+export function buildDefaultOutputPaths(startUrl, now = new Date()) {
+    const stamp = formatRunTimestamp(now);
+    const host = hostnameFromUrl(startUrl);
+    const base = `${stamp}-${host}`;
+    return {
+        rawJson: `${DEFAULT_OUTPUT_DIR}/${base}-raw.json`,
+        reportMd: `${DEFAULT_OUTPUT_DIR}/${base}-report.md`,
+        reportJson: `${DEFAULT_OUTPUT_DIR}/${base}-report.json`,
+    };
+}
+function reportBasename(inputPath) {
+    if (inputPath.endsWith("-raw.json")) {
+        return inputPath.slice(0, -"-raw.json".length);
+    }
+    if (inputPath.endsWith("-report.md")) {
+        return inputPath.slice(0, -"-report.md".length);
+    }
+    if (inputPath.endsWith("-report.json")) {
+        return inputPath.slice(0, -"-report.json".length);
+    }
+    if (inputPath.endsWith("-fix-manifest.json")) {
+        return inputPath.slice(0, -"-fix-manifest.json".length);
+    }
+    if (inputPath.endsWith(".json")) {
+        return inputPath.slice(0, -".json".length);
+    }
+    return inputPath;
+}
+export function defaultReportPathForInput(inputPath) {
+    return `${reportBasename(inputPath)}-report.md`;
+}
+export function defaultReportJsonPathForInput(inputPath) {
+    return `${reportBasename(inputPath)}-report.json`;
+}
+/** @deprecated Fix-manifest output is removed; kept for callers that still resolve paths. */
+export function defaultManifestPathForInput(inputPath) {
+    return `${reportBasename(inputPath)}-fix-manifest.json`;
+}
+/** @deprecated Agent-jobs output is removed; kept for callers that still resolve paths. */
+export function defaultAgentJobsPathForInput(inputPath) {
+    return `${reportBasename(inputPath)}-agent-jobs`;
+}

package/dist/crawler/reporter.d.ts

@@ -0,0 +1,3 @@
+import type { VerificationRecord } from "./schema.js";
+export declare function writeReportToFile(records: VerificationRecord[], outputPath: string): void;
+//# sourceMappingURL=reporter.d.ts.map

package/dist/crawler/reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/crawler/reporter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAMtD,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,kBAAkB,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CAMzF"}

package/dist/crawler/reporter.js

@@ -0,0 +1,12 @@
+import { mkdirSync, renameSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+function ensureParentDir(filePath) {
+    mkdirSync(dirname(filePath), { recursive: true });
+}
+export function writeReportToFile(records, outputPath) {
+    ensureParentDir(outputPath);
+    const tempPath = `${outputPath}.tmp`;
+    const json = JSON.stringify(records, null, 2) + "\n";
+    writeFileSync(tempPath, json, "utf8");
+    renameSync(tempPath, outputPath);
+}

package/dist/crawler/robots.d.ts

@@ -0,0 +1,5 @@
+export type RobotsPolicy = {
+    isAllowed: (url: string) => boolean;
+};
+export declare function loadRobotsPolicy(origin: string, timeoutMs: number): Promise<RobotsPolicy | null>;
+//# sourceMappingURL=robots.d.ts.map

package/dist/crawler/robots.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../src/crawler/robots.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;CACrC,CAAC;AAEF,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CA8BtG"}

package/dist/crawler/robots.js

@@ -0,0 +1,29 @@
+import robotsParser from "robots-parser";
+import { DEFAULT_USER_AGENT } from "./defaults.js";
+export async function loadRobotsPolicy(origin, timeoutMs) {
+    const createRobotsParser = robotsParser;
+    const robotsUrl = `${origin}/robots.txt`;
+    try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), timeoutMs);
+        const response = await fetch(robotsUrl, {
+            redirect: "follow",
+            signal: controller.signal,
+            headers: {
+                "user-agent": DEFAULT_USER_AGENT,
+            },
+        });
+        clearTimeout(timeout);
+        if (!response.ok) {
+            return null;
+        }
+        const body = await response.text();
+        const parsed = createRobotsParser(robotsUrl, body);
+        return {
+            isAllowed: (url) => parsed.isAllowed(url, DEFAULT_USER_AGENT) !== false,
+        };
+    }
+    catch {
+        return null;
+    }
+}

package/dist/crawler/runCrawler.d.ts

@@ -0,0 +1,22 @@
+import { type CheckGroup } from "./checkGroups.js";
+import type { VerificationRecord } from "./schema.js";
+export type RunCrawlerOptions = {
+    startUrl: string;
+    outputPath: string;
+    maxPages?: number;
+    delayMs?: number;
+    timeoutMs?: number;
+    concurrency?: number;
+    burstDelayMs?: number;
+    seedUrls?: string[];
+    excludePatterns?: string[];
+    respectRobotsTxt?: boolean;
+    activeGroups?: Set<CheckGroup>;
+    revealSecrets?: boolean;
+    quiet?: boolean;
+};
+export declare function runCrawler(options: RunCrawlerOptions): Promise<{
+    records: VerificationRecord[];
+    interrupted: boolean;
+}>;
+//# sourceMappingURL=runCrawler.d.ts.map

package/dist/crawler/runCrawler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runCrawler.d.ts","sourceRoot":"","sources":["../../src/crawler/runCrawler.ts"],"names":[],"mappings":"AAUA,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAOjE,OAAO,KAAK,EAAiB,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAKrE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,YAAY,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAAC,CAqK7H"}

package/dist/crawler/runCrawler.js

@@ -0,0 +1,172 @@
+import { CheerioCrawler, RequestQueue, log, LogLevel } from "crawlee";
+import pino from "pino";
+import { DEFAULT_BURST_DELAY_MS, DEFAULT_CONCURRENCY, DEFAULT_DELAY_MS, DEFAULT_MAX_PAGES, DEFAULT_TIMEOUT_MS, DEFAULT_USER_AGENT, } from "./defaults.js";
+import { CHECK_GROUPS } from "./checkGroups.js";
+import { extractInternalLinks, extractVerificationRecord } from "./extractors.js";
+import { detectStackFromHtml } from "./stack.js";
+import { extractSameOriginScripts } from "./jsAssets.js";
+import { loadRobotsPolicy } from "./robots.js";
+import { writeReportToFile } from "./reporter.js";
+import { scanText } from "./secrets.js";
+import { matchesExcludePattern, normalizeUrl, sleep } from "./urlUtils.js";
+const MAX_JS_ASSET_BYTES = 5_000_000; // 5 MB — skip larger bundles to bound memory
+export async function runCrawler(options) {
+    const logger = pino({ level: "info" });
+    const startUrl = normalizeUrl(options.startUrl);
+    const origin = new URL(startUrl).origin;
+    const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
+    const delayMs = options.delayMs ?? DEFAULT_DELAY_MS;
+    const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
+    const burstDelayMs = options.burstDelayMs ?? DEFAULT_BURST_DELAY_MS;
+    const excludePatterns = options.excludePatterns ?? [];
+    const seedUrls = (options.seedUrls ?? [startUrl]).map((url) => normalizeUrl(url));
+    const respectRobotsTxt = options.respectRobotsTxt ?? false;
+    const quiet = options.quiet ?? false;
+    if (quiet) {
+        log.setLevel(LogLevel.WARNING);
+    }
+    const activeGroups = options.activeGroups ?? new Set(CHECK_GROUPS);
+    const revealSecrets = options.revealSecrets ?? false;
+    const jsScanCache = new Map();
+    const failedAssets = new Set();
+    const fetchText = async (assetUrl) => {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), timeoutMs);
+        try {
+            const res = await fetch(assetUrl, {
+                signal: controller.signal,
+                headers: { "user-agent": DEFAULT_USER_AGENT },
+            });
+            if (!res.ok)
+                return null;
+            // Fast reject when the server declares an oversize body up front.
+            const contentLength = Number(res.headers.get("content-length") ?? "0");
+            if (contentLength > MAX_JS_ASSET_BYTES)
+                return null;
+            // Backstop for chunked/gzip responses that omit content-length: skip
+            // scanning bodies that turn out larger than the cap once read.
+            const text = await res.text();
+            if (text.length > MAX_JS_ASSET_BYTES)
+                return null;
+            return text;
+        }
+        catch {
+            return null;
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    };
+    const scanSecurity = async (html, pageUrl) => {
+        const secrets = [...scanText(html, "html", revealSecrets)];
+        const scripts = extractSameOriginScripts(html, pageUrl, origin);
+        let scannedAssets = 0;
+        for (const scriptUrl of scripts) {
+            if (failedAssets.has(scriptUrl))
+                continue;
+            let assetFindings = jsScanCache.get(scriptUrl);
+            if (!assetFindings) {
+                const js = await fetchText(scriptUrl);
+                if (js === null) {
+                    failedAssets.add(scriptUrl);
+                    continue;
+                }
+                assetFindings = scanText(js, scriptUrl, revealSecrets);
+                jsScanCache.set(scriptUrl, assetFindings);
+            }
+            scannedAssets += 1;
+            secrets.push(...assetFindings);
+        }
+        return { secrets, scannedAssets };
+    };
+    const robotsPolicy = respectRobotsTxt ? await loadRobotsPolicy(origin, timeoutMs) : null;
+    const requestQueue = await RequestQueue.open();
+    const records = [];
+    const seen = new Set();
+    for (const seed of seedUrls) {
+        if (!matchesExcludePattern(seed, excludePatterns)) {
+            await requestQueue.addRequest({ url: seed, uniqueKey: normalizeUrl(seed) });
+            seen.add(normalizeUrl(seed));
+        }
+    }
+    let processedSinceBurst = 0;
+    let interrupted = false;
+    const persist = () => {
+        writeReportToFile(records, options.outputPath);
+        return records;
+    };
+    process.on("SIGINT", async () => {
+        interrupted = true;
+        logger.info("[crawl] SIGINT ontvangen, graceful stop gestart");
+    });
+    logger.info(`[crawl] start url=${startUrl} output=${options.outputPath} maxPages=${maxPages} concurrency=${concurrency} delayMs=${delayMs} burstDelayMs=${burstDelayMs} timeoutMs=${timeoutMs} seeds=${seedUrls.length} respectRobotsTxt=${respectRobotsTxt}`);
+    const crawler = new CheerioCrawler({
+        requestQueue,
+        maxRequestsPerCrawl: maxPages,
+        maxConcurrency: concurrency,
+        // +60 s headroom when security scanning is active: the handler does sequential JS fetches
+        // per page; the failedAssets/jsScanCache ensures dead/cached assets aren't re-fetched.
+        requestHandlerTimeoutSecs: Math.ceil(timeoutMs / 1000) + (activeGroups.has("security") ? 60 : 0),
+        requestHandler: async ({ request, body, contentType, response }) => {
+            if (interrupted)
+                return;
+            if (delayMs > 0) {
+                await sleep(delayMs);
+            }
+            if (burstDelayMs > 0 && processedSinceBurst > 0 && processedSinceBurst % concurrency === 0) {
+                await sleep(burstDelayMs);
+            }
+            const currentUrl = normalizeUrl(request.loadedUrl ?? request.url);
+            if (robotsPolicy && !robotsPolicy.isAllowed(currentUrl)) {
+                const skipped = extractVerificationRecord("", currentUrl, null, "");
+                skipped.note = "Skipped by robots.txt policy";
+                records.push(skipped);
+                const currentRecords = persist();
+                processedSinceBurst += 1;
+                logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=robots-blocked`);
+                return;
+            }
+            const html = typeof body === "string" ? body : "";
+            const status = response?.statusCode ?? null;
+            const effectiveContentType = contentType?.type ?? "";
+            const record = extractVerificationRecord(html, currentUrl, status, effectiveContentType, activeGroups);
+            if (activeGroups.has("stack") && effectiveContentType.includes("text/html") && html) {
+                record.stack = { detections: detectStackFromHtml(html, currentUrl) };
+            }
+            if (activeGroups.has("security") && effectiveContentType.includes("text/html") && html) {
+                record.security = await scanSecurity(html, currentUrl);
+            }
+            records.push(record);
+            const currentRecords = persist();
+            processedSinceBurst += 1;
+            logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=${status ?? "error"}`);
+            if (effectiveContentType.includes("text/html")) {
+                const links = extractInternalLinks(html, currentUrl, origin);
+                for (const link of links) {
+                    const normalized = normalizeUrl(link);
+                    if (seen.has(normalized))
+                        continue;
+                    if (matchesExcludePattern(normalized, excludePatterns))
+                        continue;
+                    if (robotsPolicy && !robotsPolicy.isAllowed(normalized))
+                        continue;
+                    seen.add(normalized);
+                    await requestQueue.addRequest({ url: normalized, uniqueKey: normalized });
+                }
+            }
+        },
+        failedRequestHandler: async ({ request }, error) => {
+            const currentUrl = normalizeUrl(request.loadedUrl ?? request.url);
+            const record = extractVerificationRecord("", currentUrl, null, "");
+            record.error = error instanceof Error ? error.message : "Unknown error";
+            records.push(record);
+            const currentRecords = persist();
+            logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=error`);
+        },
+    });
+    await crawler.run();
+    const finalRecords = persist();
+    logger.info(`[crawl] klaar pages=${finalRecords.length} interrupted=${interrupted}`);
+    return { records: finalRecords, interrupted };
+}

package/dist/crawler/schema.d.ts

@@ -0,0 +1,87 @@
+export type SecretSeverity = "critical" | "review";
+export type SecretFinding = {
+    type: string;
+    provider: string;
+    severity: SecretSeverity;
+    masked: string;
+    source: string;
+};
+export type TrackingItem = {
+    provider: string;
+    id: string;
+};
+export type StackDetection = {
+    name: string;
+    categories: string[];
+    confidence: "high" | "medium" | "low";
+    evidence: string[];
+    website?: string;
+    version?: string;
+};
+export type CrawlIssueSeverity = "critical" | "error" | "warning" | "review";
+export type CrawlIssue = {
+    id: string;
+    severity: CrawlIssueSeverity;
+    message: string;
+};
+export type VerificationRecord = {
+    url: string;
+    status: number | null;
+    h1: string;
+    title: string;
+    metaTitle: string;
+    metaDescription: string;
+    robots: string;
+    template: string;
+    mainCount: number | null;
+    charset: string;
+    language: string;
+    colorScheme: string;
+    themeColor: string;
+    doctype: string;
+    viewport: string;
+    canonical: string;
+    favicon: string;
+    icons: Array<{
+        rel: string;
+        href: string;
+        type: string;
+        sizes: string;
+        media: string;
+    }>;
+    og: {
+        title: string;
+        description: string;
+        image: string;
+        url: string;
+        type: string;
+        siteName: string;
+    };
+    twitter: {
+        card: string;
+        title: string;
+        description: string;
+        image: string;
+    };
+    tracking: {
+        tagManager: TrackingItem[];
+        analytics: TrackingItem[];
+    };
+    stack?: {
+        detections: StackDetection[];
+    };
+    security: {
+        secrets: SecretFinding[];
+        scannedAssets: number;
+    };
+    images?: {
+        issues: CrawlIssue[];
+    };
+    a11yStatic?: {
+        issues: CrawlIssue[];
+    };
+    note?: string;
+    error?: string;
+};
+export declare function createEmptyVerificationRecord(url: string, status: number | null): VerificationRecord;
+//# sourceMappingURL=schema.d.ts.map