@eightmedia/flightcheck 1.1.0 → 1.2.0

package/CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Added
 
 - Findings include an optional `fix` suggestion in `*-report.json` and Markdown (`- Fix: …`)
+- Crawlee storage uses `.flightcheckStorage/<run-hash>/` and is cleaned up after each crawl
 
 ### Changed
 

package/dist/crawler/crawleeStorage.d.ts

@@ -0,0 +1,18 @@
+/** Root directory for ephemeral Crawlee request-queue storage (cwd-relative). */
+export declare const FLIGHTCHECK_STORAGE_ROOT = ".flightcheckStorage";
+/**
+ * Create a unique per-run storage path: `.flightcheckStorage/<hash>`.
+ * Concurrent runs each get their own subdirectory so cleanup of one does not
+ * wipe another run's queue files.
+ */
+export declare function createRunStoragePath(options?: {
+    cwd?: string;
+    seed?: string;
+}): {
+    rootDir: string;
+    runDir: string;
+    runId: string;
+};
+/** Remove a single run subdirectory. Ignores missing paths. */
+export declare function cleanupRunStorageDir(runDir: string): Promise<void>;
+//# sourceMappingURL=crawleeStorage.d.ts.map

package/dist/crawler/crawleeStorage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawleeStorage.d.ts","sourceRoot":"","sources":["../../src/crawler/crawleeStorage.ts"],"names":[],"mappings":"AAIA,iFAAiF;AACjF,eAAO,MAAM,wBAAwB,wBAAwB,CAAC;AAE9D;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,CAAC,EAAE;IAC7C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CASrD;AAED,+DAA+D;AAC/D,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAExE"}

package/dist/crawler/crawleeStorage.js

@@ -0,0 +1,24 @@
+import { createHash, randomBytes } from "node:crypto";
+import { rm } from "node:fs/promises";
+import path from "node:path";
+/** Root directory for ephemeral Crawlee request-queue storage (cwd-relative). */
+export const FLIGHTCHECK_STORAGE_ROOT = ".flightcheckStorage";
+/**
+ * Create a unique per-run storage path: `.flightcheckStorage/<hash>`.
+ * Concurrent runs each get their own subdirectory so cleanup of one does not
+ * wipe another run's queue files.
+ */
+export function createRunStoragePath(options) {
+    const cwd = options?.cwd ?? process.cwd();
+    const runId = createHash("sha256")
+        .update(`${options?.seed ?? ""}:${Date.now()}:${process.pid}:${randomBytes(8).toString("hex")}`)
+        .digest("hex")
+        .slice(0, 16);
+    const rootDir = path.join(cwd, FLIGHTCHECK_STORAGE_ROOT);
+    const runDir = path.join(rootDir, runId);
+    return { rootDir, runDir, runId };
+}
+/** Remove a single run subdirectory. Ignores missing paths. */
+export async function cleanupRunStorageDir(runDir) {
+    await rm(runDir, { recursive: true, force: true });
+}

package/dist/crawler/runCrawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runCrawler.d.ts","sourceRoot":"","sources":["../../src/crawler/runCrawler.ts"],"names":[],"mappings":"AAUA,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAOjE,OAAO,KAAK,EAAiB,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAMrE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,YAAY,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAAC,CA4K7H"}
+{"version":3,"file":"runCrawler.d.ts","sourceRoot":"","sources":["../../src/crawler/runCrawler.ts"],"names":[],"mappings":"AAUA,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAQjE,OAAO,KAAK,EAAiB,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAMrE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,YAAY,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAAC,CAkM7H"}

package/dist/crawler/runCrawler.js

@@ -1,7 +1,8 @@
-import { CheerioCrawler, RequestQueue, log, LogLevel } from "crawlee";
+import { CheerioCrawler, Configuration, MemoryStorage, RequestQueue, log, LogLevel } from "crawlee";
 import pino from "pino";
 import { DEFAULT_BURST_DELAY_MS, DEFAULT_CONCURRENCY, DEFAULT_DELAY_MS, DEFAULT_MAX_PAGES, DEFAULT_TIMEOUT_MS, DEFAULT_USER_AGENT, } from "./defaults.js";
 import { CHECK_GROUPS } from "./checkGroups.js";
+import { cleanupRunStorageDir, createRunStoragePath } from "./crawleeStorage.js";
 import { extractInternalLinks, extractVerificationRecord } from "./extractors.js";
 import { detectStackFromHtml } from "./stack.js";
 import { extractSameOriginScripts } from "./jsAssets.js";
@@ -82,100 +83,118 @@ export async function runCrawler(options) {
         return { secrets, scannedAssets };
     };
     const robotsPolicy = respectRobotsTxt ? await loadRobotsPolicy(origin, timeoutMs) : null;
-    const requestQueue = await RequestQueue.open();
-    const records = [];
-    const seen = new Set();
-    const discovery = createDiscoveryGate(seedUrls, maxPages);
-    // Seeds are already smart-ordered (home/root/list before details); FIFO preserves that.
-    for (const seed of seedUrls) {
-        if (!matchesExcludePattern(seed, excludePatterns)) {
-            const normalized = normalizeUrl(seed);
-            await requestQueue.addRequest({ url: seed, uniqueKey: normalized });
-            seen.add(normalized);
-            discovery.remember(normalized);
-        }
-    }
-    let processedSinceBurst = 0;
-    let interrupted = false;
-    const persist = () => {
-        writeReportToFile(records, options.outputPath);
-        return records;
-    };
-    process.on("SIGINT", async () => {
-        interrupted = true;
-        logger.info("[crawl] SIGINT ontvangen, graceful stop gestart");
+    // Per-run Crawlee storage so concurrent crawls do not share/purge each other,
+    // then remove this run's subdirectory when finished.
+    const { runDir, runId } = createRunStoragePath({ seed: startUrl });
+    const configuration = new Configuration({
+        storageClient: new MemoryStorage({ localDataDirectory: runDir }),
+        defaultRequestQueueId: runId,
+        purgeOnStart: true,
     });
-    logger.info(`[crawl] start url=${startUrl} output=${options.outputPath} maxPages=${maxPages} concurrency=${concurrency} delayMs=${delayMs} burstDelayMs=${burstDelayMs} timeoutMs=${timeoutMs} seeds=${seedUrls.length} respectRobotsTxt=${respectRobotsTxt}`);
-    const crawler = new CheerioCrawler({
-        requestQueue,
-        maxRequestsPerCrawl: maxPages,
-        maxConcurrency: concurrency,
-        // +60 s headroom when security scanning is active: the handler does sequential JS fetches
-        // per page; the failedAssets/jsScanCache ensures dead/cached assets aren't re-fetched.
-        requestHandlerTimeoutSecs: Math.ceil(timeoutMs / 1000) + (activeGroups.has("security") ? 60 : 0),
-        requestHandler: async ({ request, body, contentType, response }) => {
-            if (interrupted)
-                return;
-            if (delayMs > 0) {
-                await sleep(delayMs);
+    try {
+        const requestQueue = await RequestQueue.open(runId, { config: configuration });
+        const records = [];
+        const seen = new Set();
+        const discovery = createDiscoveryGate(seedUrls, maxPages);
+        // Seeds are already smart-ordered (home/root/list before details); FIFO preserves that.
+        for (const seed of seedUrls) {
+            if (!matchesExcludePattern(seed, excludePatterns)) {
+                const normalized = normalizeUrl(seed);
+                await requestQueue.addRequest({ url: seed, uniqueKey: normalized });
+                seen.add(normalized);
+                discovery.remember(normalized);
             }
-            if (burstDelayMs > 0 && processedSinceBurst > 0 && processedSinceBurst % concurrency === 0) {
-                await sleep(burstDelayMs);
-            }
-            const currentUrl = normalizeUrl(request.loadedUrl ?? request.url);
-            if (robotsPolicy && !robotsPolicy.isAllowed(currentUrl)) {
-                const skipped = extractVerificationRecord("", currentUrl, null, "");
-                skipped.note = "Skipped by robots.txt policy";
-                records.push(skipped);
+        }
+        let processedSinceBurst = 0;
+        let interrupted = false;
+        const persist = () => {
+            writeReportToFile(records, options.outputPath);
+            return records;
+        };
+        process.on("SIGINT", async () => {
+            interrupted = true;
+            logger.info("[crawl] SIGINT ontvangen, graceful stop gestart");
+        });
+        logger.info(`[crawl] start url=${startUrl} output=${options.outputPath} maxPages=${maxPages} concurrency=${concurrency} delayMs=${delayMs} burstDelayMs=${burstDelayMs} timeoutMs=${timeoutMs} seeds=${seedUrls.length} respectRobotsTxt=${respectRobotsTxt} storage=${runDir}`);
+        const crawler = new CheerioCrawler({
+            requestQueue,
+            maxRequestsPerCrawl: maxPages,
+            maxConcurrency: concurrency,
+            // +60 s headroom when security scanning is active: the handler does sequential JS fetches
+            // per page; the failedAssets/jsScanCache ensures dead/cached assets aren't re-fetched.
+            requestHandlerTimeoutSecs: Math.ceil(timeoutMs / 1000) + (activeGroups.has("security") ? 60 : 0),
+            requestHandler: async ({ request, body, contentType, response }) => {
+                if (interrupted)
+                    return;
+                if (delayMs > 0) {
+                    await sleep(delayMs);
+                }
+                if (burstDelayMs > 0 && processedSinceBurst > 0 && processedSinceBurst % concurrency === 0) {
+                    await sleep(burstDelayMs);
+                }
+                const currentUrl = normalizeUrl(request.loadedUrl ?? request.url);
+                if (robotsPolicy && !robotsPolicy.isAllowed(currentUrl)) {
+                    const skipped = extractVerificationRecord("", currentUrl, null, "");
+                    skipped.note = "Skipped by robots.txt policy";
+                    records.push(skipped);
+                    const currentRecords = persist();
+                    processedSinceBurst += 1;
+                    logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=robots-blocked`);
+                    return;
+                }
+                const html = typeof body === "string" ? body : "";
+                const status = response?.statusCode ?? null;
+                const effectiveContentType = contentType?.type ?? "";
+                const record = extractVerificationRecord(html, currentUrl, status, effectiveContentType, activeGroups);
+                if (activeGroups.has("stack") && effectiveContentType.includes("text/html") && html) {
+                    record.stack = { detections: detectStackFromHtml(html, currentUrl) };
+                }
+                if (activeGroups.has("security") && effectiveContentType.includes("text/html") && html) {
+                    record.security = await scanSecurity(html, currentUrl);
+                }
+                records.push(record);
                 const currentRecords = persist();
                 processedSinceBurst += 1;
-                logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=robots-blocked`);
-                return;
-            }
-            const html = typeof body === "string" ? body : "";
-            const status = response?.statusCode ?? null;
-            const effectiveContentType = contentType?.type ?? "";
-            const record = extractVerificationRecord(html, currentUrl, status, effectiveContentType, activeGroups);
-            if (activeGroups.has("stack") && effectiveContentType.includes("text/html") && html) {
-                record.stack = { detections: detectStackFromHtml(html, currentUrl) };
-            }
-            if (activeGroups.has("security") && effectiveContentType.includes("text/html") && html) {
-                record.security = await scanSecurity(html, currentUrl);
-            }
-            records.push(record);
-            const currentRecords = persist();
-            processedSinceBurst += 1;
-            logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=${status ?? "error"}`);
-            if (effectiveContentType.includes("text/html")) {
-                const links = extractInternalLinks(html, currentUrl, origin);
-                for (const link of links) {
-                    const normalized = normalizeUrl(link);
-                    if (seen.has(normalized))
-                        continue;
-                    if (matchesExcludePattern(normalized, excludePatterns))
-                        continue;
-                    if (robotsPolicy && !robotsPolicy.isAllowed(normalized))
-                        continue;
-                    if (!discovery.allowEnqueue(normalized))
-                        continue;
-                    seen.add(normalized);
-                    discovery.remember(normalized);
-                    const forefront = discovery.isStructural(normalized);
-                    await requestQueue.addRequest({ url: normalized, uniqueKey: normalized }, { forefront });
+                logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=${status ?? "error"}`);
+                if (effectiveContentType.includes("text/html")) {
+                    const links = extractInternalLinks(html, currentUrl, origin);
+                    for (const link of links) {
+                        const normalized = normalizeUrl(link);
+                        if (seen.has(normalized))
+                            continue;
+                        if (matchesExcludePattern(normalized, excludePatterns))
+                            continue;
+                        if (robotsPolicy && !robotsPolicy.isAllowed(normalized))
+                            continue;
+                        if (!discovery.allowEnqueue(normalized))
+                            continue;
+                        seen.add(normalized);
+                        discovery.remember(normalized);
+                        const forefront = discovery.isStructural(normalized);
+                        await requestQueue.addRequest({ url: normalized, uniqueKey: normalized }, { forefront });
+                    }
                 }
-            }
-        },
-        failedRequestHandler: async ({ request }, error) => {
-            const currentUrl = normalizeUrl(request.loadedUrl ?? request.url);
-            const record = extractVerificationRecord("", currentUrl, null, "");
-            record.error = error instanceof Error ? error.message : "Unknown error";
-            records.push(record);
-            const currentRecords = persist();
-            logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=error`);
-        },
-    });
-    await crawler.run();
-    const finalRecords = persist();
-    logger.info(`[crawl] klaar pages=${finalRecords.length} interrupted=${interrupted}`);
-    return { records: finalRecords, interrupted };
+            },
+            failedRequestHandler: async ({ request }, error) => {
+                const currentUrl = normalizeUrl(request.loadedUrl ?? request.url);
+                const record = extractVerificationRecord("", currentUrl, null, "");
+                record.error = error instanceof Error ? error.message : "Unknown error";
+                records.push(record);
+                const currentRecords = persist();
+                logger.info(`[crawl] processed=${currentRecords.length} url=${currentUrl} status=error`);
+            },
+        }, configuration);
+        await crawler.run();
+        const finalRecords = persist();
+        logger.info(`[crawl] klaar pages=${finalRecords.length} interrupted=${interrupted}`);
+        return { records: finalRecords, interrupted };
+    }
+    finally {
+        try {
+            await cleanupRunStorageDir(runDir);
+        }
+        catch (error) {
+            logger.warn(`[crawl] kon Crawlee storage niet opruimen path=${runDir} error=${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
 }

package/docs/crawler.md

@@ -169,3 +169,6 @@ Selection lives in `src/crawler/smartPageSelection.ts`.
 - With `--respect-robots-txt`, URLs excluded by `robots.txt` are skipped.
 - After each URL, output is written atomically.
 - On `SIGINT`, the crawl stops gracefully while preserving partial output.
+- Crawlee request-queue files go under `.flightcheckStorage/<run-hash>/` (not the generic
+  `storage/` folder). That subdirectory is deleted when the crawl finishes, so a second
+  concurrent run keeps its own files until it completes.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eightmedia/flightcheck",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Pre-launch website audit: crawl SEO/a11y/security checks and emit structured JSON + Markdown reports.",
   "license": "GPL-3.0",
   "type": "module",