@eightmedia/flightcheck 0.0.0-stage → 0.1.0

package/dist/crawler/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/crawler/schema.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,QAAQ,CAAC;AAEnD,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,cAAc,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE7E,MAAM,MAAM,UAAU,GAAG;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,KAAK,CAAC;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;IACH,EAAE,EAAE;QACF,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,OAAO,EAAE;QACP,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,QAAQ,EAAE;QACR,UAAU,EAAE,YAAY,EAAE,CAAC;QAC3B,SAAS,EAAE,YAAY,EAAE,CAAC;KAC3B,CAAC;IACF,KAAK,CAAC,EAAE;QACN,UAAU,EAAE,cAAc,EAAE,CAAC;KAC9B,CAAC;IACF,QAAQ,EAAE;QACR,OAAO,EAAE,aAAa,EAAE,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,UAAU,EAAE,CAAC;KACtB,CAAC;IACF,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,UAAU,EAAE,CAAC;KACtB,CAAC;IACF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,wBAAgB,6BAA6B,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,kBAAkB,CA2CpG"}

package/dist/crawler/schema.js

@@ -0,0 +1,44 @@
+export function createEmptyVerificationRecord(url, status) {
+    return {
+        url,
+        status,
+        h1: "",
+        title: "",
+        metaTitle: "",
+        metaDescription: "",
+        robots: "",
+        template: "",
+        mainCount: null,
+        charset: "",
+        language: "",
+        colorScheme: "",
+        themeColor: "",
+        doctype: "",
+        viewport: "",
+        canonical: "",
+        favicon: "",
+        icons: [],
+        og: {
+            title: "",
+            description: "",
+            image: "",
+            url: "",
+            type: "",
+            siteName: "",
+        },
+        twitter: {
+            card: "",
+            title: "",
+            description: "",
+            image: "",
+        },
+        tracking: {
+            tagManager: [],
+            analytics: [],
+        },
+        security: {
+            secrets: [],
+            scannedAssets: 0,
+        },
+    };
+}

package/dist/crawler/secrets.d.ts

@@ -0,0 +1,4 @@
+import type { SecretFinding } from "./schema.js";
+export declare function maskSecret(value: string): string;
+export declare function scanText(text: string, source: string, reveal?: boolean): SecretFinding[];
+//# sourceMappingURL=secrets.d.ts.map

package/dist/crawler/secrets.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"secrets.d.ts","sourceRoot":"","sources":["../../src/crawler/secrets.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAkB,MAAM,aAAa,CAAC;AA2CjE,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAKhD;AAED,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,UAAQ,GAAG,aAAa,EAAE,CA8BtF"}

package/dist/crawler/secrets.js

@@ -0,0 +1,70 @@
+const PATTERNS = [
+    { type: "google-api-key", provider: "google", severity: "review", regex: /\bAIzaSy[0-9A-Za-z_-]{33}\b/g },
+    { type: "aws-access-key-id", provider: "aws", severity: "critical", regex: /\bAKIA[0-9A-Z]{16}\b/g },
+    { type: "stripe-secret-key", provider: "stripe", severity: "critical", regex: /\bsk_live_[0-9a-zA-Z]{24,}\b/g },
+    { type: "stripe-publishable-key", provider: "stripe", severity: "review", regex: /\bpk_live_[0-9a-zA-Z]{24,}\b/g },
+    { type: "github-token", provider: "github", severity: "critical", regex: /\bgh[posru]_[0-9A-Za-z]{36,}\b/g },
+    {
+        type: "private-key",
+        provider: "generic",
+        severity: "critical",
+        regex: /-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----/g,
+        display: "[PRIVATE KEY BLOCK]",
+    },
+    {
+        type: "generic-secret",
+        provider: "generic",
+        severity: "critical",
+        regex: /["']?(?:secret|password|api[_-]?secret)["']?\s*[:=]\s*["']([^"'\s]{8,})["']/gi,
+        group: 1,
+        generic: true,
+    },
+    {
+        type: "generic-api-key",
+        provider: "generic",
+        severity: "review",
+        regex: /["']?(?:api[_-]?key|access[_-]?token|auth[_-]?token)["']?\s*[:=]\s*["']([^"'\s]{12,})["']/gi,
+        group: 1,
+        generic: true,
+    },
+];
+export function maskSecret(value) {
+    if (value.length <= 10) {
+        return `${value.slice(0, 2)}…${value.slice(-2)}`;
+    }
+    return `${value.slice(0, 6)}…${value.slice(-4)}`;
+}
+export function scanText(text, source, reveal = false) {
+    const findings = [];
+    const seen = new Set(); // dedup: type + raw value
+    const specificValues = new Set(); // raw values from non-generic patterns
+    const collect = (pattern) => {
+        const re = new RegExp(pattern.regex.source, pattern.regex.flags);
+        let match;
+        while ((match = re.exec(text)) !== null) {
+            const raw = pattern.group ? match[pattern.group] : match[0];
+            if (!raw)
+                continue;
+            if (pattern.generic && specificValues.has(raw))
+                continue;
+            const key = `${pattern.type}:${raw}`;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            if (!pattern.generic)
+                specificValues.add(raw);
+            findings.push({
+                type: pattern.type,
+                provider: pattern.provider,
+                severity: pattern.severity,
+                masked: pattern.display ?? (reveal ? raw : maskSecret(raw)),
+                source,
+            });
+        }
+    };
+    for (const pattern of PATTERNS.filter((p) => !p.generic))
+        collect(pattern);
+    for (const pattern of PATTERNS.filter((p) => p.generic))
+        collect(pattern);
+    return findings;
+}

package/dist/crawler/sitemap.d.ts

@@ -0,0 +1,8 @@
+export type SitemapDocumentType = "index" | "urlset" | "unknown";
+export declare function detectSitemapDocumentType(xml: string): SitemapDocumentType;
+export declare function extractSitemapLocValues(xml: string): string[];
+export declare function canonicalSiteHostname(hostname: string): string;
+export declare function isRelatedSiteUrl(url: string, siteOrigin: string): boolean;
+export declare function processSitemapLocations(xml: string, siteOrigin: string, visitedSitemaps: Set<string>, pageUrls: Set<string>, sitemapQueue: string[]): void;
+export declare function collectUrlsFromSitemap(sitemapUrl: string, origin: string, timeoutMs: number): Promise<Set<string>>;
+//# sourceMappingURL=sitemap.d.ts.map

package/dist/crawler/sitemap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/crawler/sitemap.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEjE,wBAAgB,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,mBAAmB,CAI1E;AAED,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAW7D;AAED,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAE9D;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAQzE;AAWD,wBAAgB,uBAAuB,CACrC,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,EAClB,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,EAC5B,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,EACrB,YAAY,EAAE,MAAM,EAAE,GACrB,IAAI,CA8BN;AAED,wBAAsB,sBAAsB,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAmCxH"}

package/dist/crawler/sitemap.js

@@ -0,0 +1,105 @@
+import { DEFAULT_MAX_SITEMAPS, DEFAULT_MAX_SITEMAP_URLS, DEFAULT_USER_AGENT } from "./defaults.js";
+import { normalizeUrl } from "./urlUtils.js";
+export function detectSitemapDocumentType(xml) {
+    if (/<sitemapindex[\s>]/i.test(xml))
+        return "index";
+    if (/<urlset[\s>]/i.test(xml))
+        return "urlset";
+    return "unknown";
+}
+export function extractSitemapLocValues(xml) {
+    const values = [];
+    const locRegex = /<loc>\s*([\s\S]*?)\s*<\/loc>/gi;
+    let match;
+    while ((match = locRegex.exec(xml)) !== null) {
+        const locValue = (match[1] ?? "").trim();
+        if (locValue)
+            values.push(locValue);
+    }
+    return values;
+}
+export function canonicalSiteHostname(hostname) {
+    return hostname.toLowerCase().startsWith("www.") ? hostname.slice(4).toLowerCase() : hostname.toLowerCase();
+}
+export function isRelatedSiteUrl(url, siteOrigin) {
+    try {
+        const parsed = new URL(normalizeUrl(url));
+        const site = new URL(siteOrigin);
+        return canonicalSiteHostname(parsed.hostname) === canonicalSiteHostname(site.hostname);
+    }
+    catch {
+        return false;
+    }
+}
+function looksLikeSitemapFile(url) {
+    try {
+        const pathname = new URL(url).pathname.toLowerCase();
+        return pathname.endsWith(".xml") && pathname.includes("sitemap");
+    }
+    catch {
+        return false;
+    }
+}
+export function processSitemapLocations(xml, siteOrigin, visitedSitemaps, pageUrls, sitemapQueue) {
+    const docType = detectSitemapDocumentType(xml);
+    const locs = extractSitemapLocValues(xml);
+    for (const locValue of locs) {
+        try {
+            const normalizedLoc = normalizeUrl(locValue);
+            if (!isRelatedSiteUrl(normalizedLoc, siteOrigin))
+                continue;
+            if (docType === "index") {
+                if (!visitedSitemaps.has(normalizedLoc)) {
+                    sitemapQueue.push(normalizedLoc);
+                }
+                continue;
+            }
+            if (docType === "urlset") {
+                pageUrls.add(normalizedLoc);
+                continue;
+            }
+            if (looksLikeSitemapFile(normalizedLoc) && !visitedSitemaps.has(normalizedLoc)) {
+                sitemapQueue.push(normalizedLoc);
+            }
+            else {
+                pageUrls.add(normalizedLoc);
+            }
+        }
+        catch {
+            // Ignore malformed loc values.
+        }
+    }
+}
+export async function collectUrlsFromSitemap(sitemapUrl, origin, timeoutMs) {
+    const urls = new Set();
+    const visitedSitemaps = new Set();
+    const queue = [normalizeUrl(sitemapUrl)];
+    while (queue.length > 0 &&
+        visitedSitemaps.size < DEFAULT_MAX_SITEMAPS &&
+        urls.size < DEFAULT_MAX_SITEMAP_URLS) {
+        const currentSitemap = queue.shift();
+        if (!currentSitemap || visitedSitemaps.has(currentSitemap))
+            continue;
+        visitedSitemaps.add(currentSitemap);
+        try {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), timeoutMs);
+            const response = await fetch(currentSitemap, {
+                redirect: "follow",
+                signal: controller.signal,
+                headers: {
+                    "user-agent": DEFAULT_USER_AGENT,
+                },
+            });
+            clearTimeout(timeout);
+            if (!response.ok)
+                continue;
+            const text = await response.text();
+            processSitemapLocations(text, origin, visitedSitemaps, urls, queue);
+        }
+        catch {
+            // Ignore sitemap fetch errors and continue.
+        }
+    }
+    return urls;
+}

package/dist/crawler/stack.d.ts

@@ -0,0 +1,3 @@
+export type { StackCategory, StackDetection } from "./stackDetect.js";
+export { CURATED_STACK_NAMES, detectStackFromDns, detectStackFromHeaders, detectStackFromHtml, detectStackFromRobots, isCuratedStackName, } from "./stackDetect.js";
+//# sourceMappingURL=stack.d.ts.map

package/dist/crawler/stack.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stack.d.ts","sourceRoot":"","sources":["../../src/crawler/stack.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC"}

package/dist/crawler/stack.js

@@ -0,0 +1 @@
+export { CURATED_STACK_NAMES, detectStackFromDns, detectStackFromHeaders, detectStackFromHtml, detectStackFromRobots, isCuratedStackName, } from "./stackDetect.js";

package/dist/crawler/stackDetect.d.ts

@@ -0,0 +1,24 @@
+export type StackCategory = "Platform" | "Framework" | "CMS" | "Search" | "Hosting" | "CDN";
+export type StackDetection = {
+    name: string;
+    categories: string[];
+    confidence: "high" | "medium" | "low";
+    evidence: string[];
+    website?: string;
+    version?: string;
+    detector?: "flightcheck-v1";
+};
+export declare function detectStackFromHtml(html: string, pageUrl: string): StackDetection[];
+export declare function detectStackFromHeaders(headers: Record<string, string>): StackDetection[];
+export declare function detectStackFromDns(records: {
+    cname: string[];
+    ns: string[];
+    mx: Array<{
+        exchange: string;
+    }>;
+    txt: string[];
+}): StackDetection[];
+export declare function detectStackFromRobots(_body: string): StackDetection[];
+export declare const CURATED_STACK_NAMES: Set<string>;
+export declare function isCuratedStackName(name: string): boolean;
+//# sourceMappingURL=stackDetect.d.ts.map

package/dist/crawler/stackDetect.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stackDetect.d.ts","sourceRoot":"","sources":["../../src/crawler/stackDetect.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,SAAS,GAAG,KAAK,CAAC;AAE5F,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B,CAAC;AA4gBF,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,cAAc,EAAE,CAgEnF;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,cAAc,EAAE,CAWxF;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE;IAC1C,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,EAAE,EAAE,MAAM,EAAE,CAAC;IACb,EAAE,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChC,GAAG,EAAE,MAAM,EAAE,CAAC;CACf,GAAG,cAAc,EAAE,CAgBnB;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE,CAErE;AAED,eAAO,MAAM,mBAAmB,aAS9B,CAAC;AAEH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAExD"}