@fulldotdev/scan 0.1.0

package/dist/engine/srcset.d.ts

@@ -0,0 +1 @@
+export declare function srcsetUrls(value: string): string[];

package/dist/engine/srcset.js

@@ -0,0 +1,31 @@
+// URLs in srcset may contain commas (for example image-CDN parameters).
+// A URL runs until whitespace; trailing commas terminate a bare candidate.
+export function srcsetUrls(value) {
+    const urls = [];
+    let at = 0;
+    const space = (c) => /[\t\n\f\r ]/.test(c);
+    while (at < value.length) {
+        while (at < value.length && (space(value[at]) || value[at] === ","))
+            at++;
+        const start = at;
+        while (at < value.length && !space(value[at]))
+            at++;
+        const token = value.slice(start, at);
+        if (!token)
+            break;
+        urls.push(token.replace(/,+$/, ""));
+        if (token.endsWith(","))
+            continue;
+        let parentheses = 0;
+        while (at < value.length) {
+            const c = value[at++];
+            if (c === "(")
+                parentheses++;
+            if (c === ")")
+                parentheses = Math.max(0, parentheses - 1);
+            if (c === "," && parentheses === 0)
+                break;
+        }
+    }
+    return urls.filter(Boolean);
+}

package/dist/engine/state.d.ts

@@ -0,0 +1,30 @@
+import type { Artifact, Job, JobKind, JobResult, Observation, Scan, UrlRow } from "./types.js";
+export interface StoredArtifact extends Artifact {
+    body: Buffer;
+    sha256: string;
+    bytes: number;
+}
+export declare class ScanState {
+    readonly scan: Scan;
+    readonly observations: Map<string, Observation>;
+    readonly artifacts: Map<string, StoredArtifact>;
+    readonly urls: Map<string, UrlRow>;
+    readonly jobs: Job[];
+    private readonly jobIndex;
+    constructor(scan: Scan);
+    enqueue(kind: JobKind, name: string): void;
+    next(kinds: JobKind[]): Job | undefined;
+    pending(kinds: JobKind[]): number;
+    cancelQueued(): number;
+    jobCounts(): {
+        kind: string;
+        status: string;
+        count: number;
+    }[];
+    get(kind: string, name: string): Observation | undefined;
+    records(kinds?: string[]): Observation[];
+    urlRows(): UrlRow[];
+    hasUsableMarkdown(): boolean;
+    record(result: JobResult): void;
+    twinIsPage(url: string): boolean;
+}

package/dist/engine/state.js

@@ -0,0 +1,198 @@
+import { createHash } from "node:crypto";
+import { isPaginationQuery, scopeRule } from "./crawl-scope.js";
+import { slashTwin } from "./network.js";
+const key = (kind, name) => `${kind}\0${name}`;
+// Everything one scan knows, in memory: observations by kind and key,
+// artifacts, the discovered URLs with why they were or were not followed,
+// and the job queue. Collectors return results; `record` admits them and
+// turns candidates into new jobs under the crawl-scope rules.
+export class ScanState {
+    scan;
+    observations = new Map();
+    artifacts = new Map();
+    urls = new Map();
+    jobs = [];
+    jobIndex = new Map();
+    constructor(scan) {
+        this.scan = scan;
+    }
+    enqueue(kind, name) {
+        const id = key(kind, name);
+        if (this.jobIndex.has(id))
+            return;
+        const job = { kind, key: name, status: "queued", attempts: 0 };
+        this.jobs.push(job);
+        this.jobIndex.set(id, job);
+    }
+    next(kinds) {
+        for (const kind of kinds) {
+            const job = this.jobs.find((j) => j.kind === kind && j.status === "queued");
+            if (job)
+                return job;
+        }
+        return undefined;
+    }
+    pending(kinds) {
+        return this.jobs.filter((j) => kinds.includes(j.kind) && j.status === "queued").length;
+    }
+    cancelQueued() {
+        let count = 0;
+        for (const job of this.jobs)
+            if (job.status === "queued") {
+                job.status = "cancelled";
+                count++;
+            }
+        return count;
+    }
+    jobCounts() {
+        const counts = new Map();
+        for (const job of this.jobs) {
+            const id = `${job.kind}\0${job.status}`;
+            counts.set(id, (counts.get(id) ?? 0) + 1);
+        }
+        return [...counts].map(([id, count]) => {
+            const [kind, status] = id.split("\0");
+            return { kind, status, count };
+        });
+    }
+    get(kind, name) {
+        return this.observations.get(key(kind, name));
+    }
+    records(kinds) {
+        const all = [...this.observations.values()];
+        return kinds ? all.filter((o) => kinds.includes(o.kind)) : all;
+    }
+    urlRows() {
+        return [...this.urls.values()];
+    }
+    hasUsableMarkdown() {
+        return this.records(["markdown"]).some((m) => m.data?.usable);
+    }
+    record(result) {
+        const { scan } = this;
+        for (const item of result.observations)
+            this.observations.set(key(item.kind, item.key), item);
+        for (const item of result.artifacts ?? []) {
+            const body = Buffer.isBuffer(item.body)
+                ? item.body
+                : Buffer.from(item.body);
+            this.artifacts.set(key(item.kind, item.key), {
+                ...item,
+                body,
+                sha256: createHash("sha256").update(body).digest("hex"),
+                bytes: body.length,
+            });
+        }
+        // Twins and HTML destinations this job already fetched need no job.
+        const observedResources = new Set(result.observations
+            .filter((o) => o.kind === "resource")
+            .map((o) => o.key));
+        const scheduled = {
+            page: 0,
+            resource: 0,
+            sitemap: 0,
+        };
+        for (const row of this.urls.values())
+            if (row.scheduled)
+                scheduled[row.kind]++;
+        let total = this.urls.size;
+        const grouped = new Map();
+        for (const candidate of result.candidates ?? []) {
+            const id = key(candidate.kind, candidate.url);
+            const row = grouped.get(id) ?? {
+                url: candidate.url,
+                kind: candidate.kind,
+                sources: new Set(),
+            };
+            row.sources.add(candidate.source);
+            grouped.set(id, row);
+        }
+        // Query-string variants per path that are not pagination: the existing
+        // count decides whether another variant is still followed.
+        const variantCounts = new Map();
+        for (const row of this.urls.values())
+            if (row.kind === "page" &&
+                row.scheduled &&
+                row.url.includes("?") &&
+                !isPaginationQuery(row.url)) {
+                const path = row.url.split("?")[0];
+                variantCounts.set(path, (variantCounts.get(path) ?? 0) + 1);
+            }
+        const limits = {
+            page: scan.options.maxPages ?? Infinity,
+            resource: scan.options.maxResources ?? Infinity,
+            sitemap: scan.options.maxSitemaps ?? Infinity,
+        };
+        for (const [id, candidate] of grouped) {
+            const previous = this.urls.get(id);
+            if (previous) {
+                for (const source of candidate.sources)
+                    previous.sources.push(source);
+                previous.sources = [...new Set(previous.sources)];
+                continue;
+            }
+            if (scan.options.maxDiscoveredUrls !== undefined &&
+                total >= scan.options.maxDiscoveredUrls) {
+                scan.coverage.omittedCandidates++;
+                continue;
+            }
+            let skip = null;
+            const rule = candidate.kind === "page" ? scopeRule(candidate.url) : null;
+            if (rule)
+                skip = rule;
+            else if (candidate.kind === "page" &&
+                new URL(candidate.url).search &&
+                !isPaginationQuery(candidate.url)) {
+                const path = candidate.url.split("?")[0];
+                const n = variantCounts.get(path) ?? 0;
+                if (n >= (scan.options.maxQueryVariantsPerPath ?? Infinity))
+                    skip = "query-variants-limited";
+                else
+                    variantCounts.set(path, n + 1);
+            }
+            if (!skip &&
+                candidate.kind === "resource" &&
+                (observedResources.has(candidate.url) ||
+                    this.get("resource", candidate.url)))
+                skip = "observed-by-page-job";
+            else if (candidate.kind === "resource" &&
+                (this.urls.has(key("page", candidate.url)) ||
+                    grouped.has(key("page", candidate.url))))
+                skip = "crawled-as-page";
+            else if (!skip && scheduled[candidate.kind] >= limits[candidate.kind])
+                skip = "explicit-limit";
+            const shouldSchedule = !skip;
+            total++;
+            this.urls.set(id, {
+                url: candidate.url,
+                kind: candidate.kind,
+                sources: [...candidate.sources],
+                scheduled: shouldSchedule,
+                skipReason: skip,
+            });
+            if (shouldSchedule) {
+                scheduled[candidate.kind]++;
+                this.enqueue(candidate.kind, candidate.url);
+            }
+        }
+        // Every HTML page gets the browser pass unless explicitly limited.
+        if (scan.options.browser) {
+            const limit = scan.options.maxBrowserPages ?? Infinity;
+            let browserJobs = this.jobs.filter((j) => j.kind === "browser").length;
+            for (const page of result.observations)
+                if (page.kind === "page" &&
+                    page.data?.html !== false &&
+                    page.data?.http?.outcome === "ok" &&
+                    browserJobs < limit) {
+                    browserJobs++;
+                    this.enqueue("browser", page.key);
+                }
+        }
+    }
+    // The trailing-slash twin of a page is only a resource when the twin was
+    // not crawled as a page itself.
+    twinIsPage(url) {
+        const twin = slashTwin(url);
+        return !!twin && this.observations.has(key("page", twin));
+    }
+}

package/dist/engine/structured-data.d.ts

@@ -0,0 +1,83 @@
+export interface Entity {
+    type: string;
+    types: string[];
+    node: Record<string, any>;
+    path: string;
+}
+export declare function entities(structuredData: {
+    parseValid?: boolean;
+    data?: any;
+}[]): Entity[];
+export declare function richResultChecks(found: Entity[]): {
+    type: string;
+    result: string;
+    path: string;
+    missingRequired: string[];
+    missingRecommended: string[];
+    problems: string[];
+    scope: string;
+}[];
+export interface OfferFacts {
+    name: string | null;
+    offers: {
+        price: number | null;
+        priceCurrency: string | null;
+        availability: string | null;
+        raw: any;
+    }[];
+    aggregate: {
+        low: number | null;
+        high: number | null;
+        currency: string | null;
+    } | null;
+    path: string;
+}
+export declare function offerFacts(found: Entity[]): OfferFacts[];
+export type Outcome = "match" | "mismatch" | "review" | "unknown";
+export declare function compareOffers(products: OfferFacts[], page: {
+    prices: {
+        currency: string;
+        amount: number | null;
+    }[];
+    availability: {
+        inStock: boolean;
+        outOfStock: boolean;
+        inStockText: string | null;
+        outOfStockText: string | null;
+    };
+    headings: {
+        level: number;
+        text: string;
+    }[];
+    title: string;
+}): {
+    products: {
+        name: string | null;
+        path: string;
+        declared: {
+            price: number;
+            prices: number[];
+            range: {
+                low: number | null;
+                high: number | null;
+                currency: string | null;
+            } | null;
+            currency: string | null;
+            availability: string | null;
+        };
+        visible: {
+            prices: number[];
+            currencies: string[];
+            availability: string | null;
+            h1: string;
+        };
+        ambiguous: boolean;
+        outcomes: {
+            price: Outcome;
+            currency: Outcome;
+            availability: Outcome;
+            name: "match" | "review" | "unknown";
+        };
+    }[];
+    scope: string;
+} | null;

package/dist/engine/structured-data.js

@@ -0,0 +1,331 @@
+// JSON-LD entities, the properties Google documents as required for the
+// rich results Scan recognises, and the comparison between declared
+// offers and what the page visibly shows. Presence checks only: nothing
+// here establishes rich-result eligibility.
+const asArray = (v) => v === undefined || v === null ? [] : Array.isArray(v) ? v : [v];
+export function entities(structuredData) {
+    const found = [];
+    const walk = (node, path, depth) => {
+        if (depth > 8 || !node)
+            return;
+        if (Array.isArray(node)) {
+            node.forEach((n, i) => walk(n, `${path}[${i}]`, depth + 1));
+            return;
+        }
+        if (typeof node !== "object")
+            return;
+        const types = asArray(node["@type"]).filter((t) => typeof t === "string");
+        if (types.length)
+            found.push({ type: types[0], types, node, path });
+        for (const [key, value] of Object.entries(node))
+            if (key !== "@type" && typeof value === "object")
+                walk(value, `${path}.${key}`, depth + 1);
+    };
+    structuredData.forEach((block, i) => {
+        if (block.parseValid && block.data)
+            walk(block.data, `$[${i}]`, 0);
+    });
+    return found;
+}
+// Properties documented as required by Google's structured-data guides for
+// each result type. Recommended properties are listed separately and only
+// ever produce informational evidence.
+const richResults = {
+    Product: {
+        required: ["name"],
+        recommended: ["image", "description", "offers", "sku", "brand"],
+        result: "Product snippet / merchant listing",
+    },
+    Offer: {
+        required: ["price|priceSpecification", "priceCurrency"],
+        recommended: ["availability", "url", "itemCondition"],
+        result: "Product offer",
+    },
+    AggregateOffer: {
+        required: ["lowPrice", "priceCurrency"],
+        recommended: ["highPrice", "offerCount"],
+        result: "Product offer range",
+    },
+    Article: {
+        required: ["headline"],
+        recommended: ["image", "author", "datePublished", "dateModified"],
+        result: "Article",
+    },
+    NewsArticle: {
+        required: ["headline"],
+        recommended: ["image", "author", "datePublished", "dateModified"],
+        result: "Article",
+    },
+    BlogPosting: {
+        required: ["headline"],
+        recommended: ["image", "author", "datePublished", "dateModified"],
+        result: "Article",
+    },
+    Organization: {
+        required: ["name"],
+        recommended: ["url", "logo", "sameAs", "contactPoint", "address"],
+        result: "Organization",
+    },
+    LocalBusiness: {
+        required: ["name", "address"],
+        recommended: ["telephone", "openingHoursSpecification", "url", "geo"],
+        result: "Local business",
+    },
+    BreadcrumbList: {
+        required: ["itemListElement"],
+        recommended: [],
+        result: "Breadcrumb",
+    },
+    FAQPage: { required: ["mainEntity"], recommended: [], result: "FAQ" },
+    Question: {
+        required: ["name", "acceptedAnswer"],
+        recommended: [],
+        result: "FAQ question",
+    },
+    Event: {
+        required: ["name", "startDate", "location"],
+        recommended: ["endDate", "image", "description", "offers"],
+        result: "Event",
+    },
+    Recipe: {
+        required: ["name", "image"],
+        recommended: ["author", "datePublished", "recipeIngredient"],
+        result: "Recipe",
+    },
+    JobPosting: {
+        required: [
+            "title",
+            "description",
+            "datePosted",
+            "hiringOrganization",
+            "jobLocation",
+        ],
+        recommended: ["validThrough", "employmentType", "baseSalary"],
+        result: "Job posting",
+    },
+    VideoObject: {
+        required: ["name", "thumbnailUrl", "uploadDate"],
+        recommended: ["description", "contentUrl", "duration"],
+        result: "Video",
+    },
+    Review: {
+        required: ["itemReviewed", "reviewRating", "author"],
+        recommended: [],
+        result: "Review snippet",
+    },
+    WebSite: {
+        required: ["name", "url"],
+        recommended: ["potentialAction"],
+        result: "Site name / sitelinks search box",
+    },
+};
+export function richResultChecks(found) {
+    return found.flatMap((entity) => {
+        const spec = richResults[entity.type];
+        if (!spec)
+            return [];
+        const has = (property) => property
+            .split("|")
+            .some((p) => entity.node[p] !== undefined &&
+            entity.node[p] !== null &&
+            entity.node[p] !== "");
+        const missingRequired = spec.required.filter((p) => !has(p));
+        const missingRecommended = spec.recommended.filter((p) => !has(p));
+        const problems = [];
+        if (entity.type === "BreadcrumbList") {
+            const items = asArray(entity.node.itemListElement);
+            items.forEach((item, i) => {
+                if (item?.position === undefined)
+                    problems.push(`item ${i + 1} has no position`);
+                if (!item?.name && !item?.item?.name)
+                    problems.push(`item ${i + 1} has no name`);
+                if (i < items.length - 1 && !item?.item)
+                    problems.push(`item ${i + 1} has no item URL`);
+            });
+        }
+        if (entity.type === "FAQPage") {
+            const questions = asArray(entity.node.mainEntity);
+            if (!questions.length)
+                problems.push("mainEntity is empty");
+        }
+        return [
+            {
+                type: entity.type,
+                result: spec.result,
+                path: entity.path,
+                missingRequired,
+                missingRecommended,
+                problems,
+                scope: "Presence of properties Google documents for this result type; not a validation of values or eligibility",
+            },
+        ];
+    });
+}
+const availabilityValue = (value) => String(value ?? "")
+    .replace(/^https?:\/\/schema\.org\//i, "")
+    .replace(/^schema:/i, "");
+export function offerFacts(found) {
+    const number = (v) => {
+        if (typeof v === "number")
+            return v;
+        if (typeof v === "string") {
+            const n = Number(v.replace(/[^\d.,-]/g, "").replace(",", "."));
+            return Number.isFinite(n) ? n : null;
+        }
+        return null;
+    };
+    return found
+        .filter((e) => e.types.some((t) => /^Product(Group)?$/.test(t)))
+        .map((product) => {
+        const offers = asArray(product.node.offers).flatMap((offer) => offer && typeof offer === "object" ? [offer] : []);
+        const aggregate = offers.find((o) => asArray(o["@type"]).includes("AggregateOffer"));
+        const single = offers.filter((o) => !asArray(o["@type"]).includes("AggregateOffer"));
+        const nested = aggregate ? asArray(aggregate.offers) : [];
+        return {
+            name: typeof product.node.name === "string" ? product.node.name : null,
+            offers: [...single, ...nested].map((o) => ({
+                price: number(o.price ??
+                    o.priceSpecification?.price ??
+                    asArray(o.priceSpecification)[0]?.price),
+                priceCurrency: o.priceCurrency ??
+                    o.priceSpecification?.priceCurrency ??
+                    asArray(o.priceSpecification)[0]?.priceCurrency ??
+                    null,
+                availability: o.availability
+                    ? availabilityValue(o.availability)
+                    : null,
+                raw: {
+                    price: o.price,
+                    priceCurrency: o.priceCurrency,
+                    availability: o.availability,
+                },
+            })),
+            aggregate: aggregate
+                ? {
+                    low: number(aggregate.lowPrice),
+                    high: number(aggregate.highPrice),
+                    currency: aggregate.priceCurrency ?? null,
+                }
+                : null,
+            path: product.path,
+        };
+    });
+}
+// Declared offers against the visible page. Several products, price ranges
+// or many visible prices make the comparison ambiguous, which is reported as
+// review, never as a mismatch.
+export function compareOffers(products, page) {
+    if (!products.length)
+        return null;
+    const visibleAmounts = [
+        ...new Set(page.prices.map((p) => p.amount).filter((a) => a !== null)),
+    ];
+    const visibleCurrencies = [...new Set(page.prices.map((p) => p.currency))];
+    const normalize = (s) => s
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s]/gu, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+    const h1 = page.headings.find((h) => h.level === 1)?.text ?? "";
+    const results = products.map((product) => {
+        const offers = product.offers;
+        const distinctPrices = [
+            ...new Set(offers.map((o) => o.price).filter((p) => p !== null)),
+        ];
+        const ambiguous = products.length > 1 ||
+            !!product.aggregate ||
+            distinctPrices.length > 1 ||
+            visibleAmounts.length > 6;
+        const declaredPrice = distinctPrices[0] ?? product.aggregate?.low ?? null;
+        let price = "unknown";
+        if (declaredPrice !== null && visibleAmounts.length) {
+            const shown = visibleAmounts.some((a) => Math.abs(a - declaredPrice) < 0.005);
+            price = shown ? "match" : ambiguous ? "review" : "mismatch";
+        }
+        else if (declaredPrice !== null || visibleAmounts.length)
+            price = "review";
+        const declaredCurrency = offers.find((o) => o.priceCurrency)?.priceCurrency ??
+            product.aggregate?.currency ??
+            null;
+        let currency = "unknown";
+        if (declaredCurrency && visibleCurrencies.length) {
+            const matches = visibleCurrencies.some((c) => c === declaredCurrency || c.split("/").includes(declaredCurrency));
+            currency = visibleCurrencies.includes("$")
+                ? "review"
+                : matches
+                    ? visibleCurrencies.length === 1
+                        ? "match"
+                        : "review"
+                    : visibleCurrencies.length === 1 && !ambiguous
+                        ? "mismatch"
+                        : "review";
+        }
+        // Several offers (variants) or both stock phrasings on one page (a sold
+        // out variant, related products) cannot be matched to one declaration
+        // from page text alone: review, never a mismatch.
+        const declaredAvailabilities = [
+            ...new Set(offers.map((o) => o.availability).filter(Boolean)),
+        ];
+        const declaredAvailability = declaredAvailabilities[0] ?? null;
+        let availability = "unknown";
+        if (declaredAvailability) {
+            const says = /InStock|InStoreOnly|OnlineOnly|LimitedAvailability|PreOrder|BackOrder/i.test(declaredAvailability)
+                ? "in"
+                : /OutOfStock|SoldOut|Discontinued/i.test(declaredAvailability)
+                    ? "out"
+                    : null;
+            const mixedVisible = page.availability.outOfStock && page.availability.inStock;
+            const visible = mixedVisible
+                ? "mixed"
+                : page.availability.outOfStock
+                    ? "out"
+                    : page.availability.inStock
+                        ? "in"
+                        : null;
+            if (!says || !visible)
+                availability = "unknown";
+            else if (visible === "mixed" ||
+                offers.length > 1 ||
+                declaredAvailabilities.length > 1)
+                availability = says === visible ? "match" : "review";
+            else if (says === visible)
+                availability = "match";
+            else
+                availability = ambiguous ? "review" : "mismatch";
+        }
+        let name = "unknown";
+        if (product.name) {
+            const declared = normalize(product.name);
+            const visibleNames = [h1, page.title].map(normalize).filter(Boolean);
+            name = visibleNames.some((v) => v.includes(declared) || declared.includes(v))
+                ? "match"
+                : "review";
+        }
+        return {
+            name: product.name,
+            path: product.path,
+            declared: {
+                price: declaredPrice,
+                prices: distinctPrices,
+                range: product.aggregate,
+                currency: declaredCurrency,
+                availability: declaredAvailability,
+            },
+            visible: {
+                prices: visibleAmounts.slice(0, 20),
+                currencies: visibleCurrencies,
+                availability: page.availability.outOfStock && page.availability.inStock
+                    ? `${page.availability.inStockText} and ${page.availability.outOfStockText}`
+                    : (page.availability.outOfStockText ??
+                        page.availability.inStockText),
+                h1,
+            },
+            ambiguous,
+            outcomes: { price, currency, availability, name },
+        };
+    });
+    return {
+        products: results,
+        scope: "Visible prices and stock wording in the main text compared with JSON-LD offers; several products, price ranges or many visible prices are reported for review rather than as mismatches",
+    };
+}