@seostudio/audit-engine 0.0.0-stage → 0.1.0

package/dist/host-gate.d.ts

@@ -0,0 +1,68 @@
+/**
+ * ONE PACE PER HOST, whoever crawls it: every request the platform's crawler sends to a site first
+ * takes the host's next free slot, shared across every project, payer and process (Redis on the
+ * server, through a gate the host lends), so many accounts pointed at one site never multiply the rate it is asked at. A host that
+ * answered 429 or 503 is backed off for every crawl of it at once. The member's own computer paces
+ * itself and never takes a slot.
+ *
+ * A slot is a reservation: the gate answers how long the caller must wait, and moves the host's next
+ * slot one interval past it, in one atomic step.
+ */
+/** A host's shared pace. */
+interface SeoHostGate {
+    /**
+     * Reserves the host's next slot.
+     *
+     * @param host - The host, `www.` dropped.
+     * @param intervalMs - The least time between two requests to it (its robots.txt `Crawl-delay` or our floor).
+     * @param now - Now, in ms.
+     * @returns How long to wait before the request, in ms (0 when it may go now).
+     */
+    reserve: (host: string, intervalMs: number, now: number) => Promise<number>;
+    /**
+     * Leaves a host alone until a moment, for every crawl of it (never brings its next slot nearer).
+     *
+     * @param host - The host, `www.` dropped.
+     * @param untilMs - When it may be asked again, in ms.
+     * @param now - Now, in ms.
+     */
+    backOff: (host: string, untilMs: number, now: number) => Promise<void>;
+}
+/**
+ * The key a host's pace is kept under: the host lowercased, its `www.` and trailing dot dropped, so a
+ * site's two spellings share one pace.
+ *
+ * @param host - The host.
+ * @returns The key.
+ */
+declare function hostGateKey(host: string): string;
+/**
+ * Whether a host is one of the shared CDN hosts ({@link SEO_SHARED_CDN_HOSTS}): its exact name or a
+ * subdomain of one listed.
+ *
+ * @param host - The host.
+ * @returns Whether it is.
+ */
+declare function isSharedCdnHost(host: string): boolean;
+/**
+ * A gate held in this process's memory: the same rules as the shared one, for one process (and tests).
+ *
+ * @returns The gate.
+ */
+declare function memoryHostGate(): SeoHostGate;
+/**
+ * How long a 429 or 503 answer asks the crawler to stay away: its `Retry-After` (seconds or an HTTP
+ * date), else the default, never past the ceiling.
+ *
+ * @param retryAfter - The header, or null.
+ * @param now - Now, in ms.
+ * @param defaults - The default and the ceiling, in ms.
+ * @returns The pause, in ms.
+ */
+declare function retryAfterMs(retryAfter: string | null, now: number, defaults: {
+    defaultMs: number;
+    maxMs: number;
+}): number;
+
+export { hostGateKey, isSharedCdnHost, memoryHostGate, retryAfterMs };
+export type { SeoHostGate };

package/dist/host-gate.js

@@ -0,0 +1 @@
+export { hostGateKey, isSharedCdnHost, memoryHostGate, retryAfterMs } from './chunk-6QPF6O7D.js';

package/dist/index.d.ts

@@ -0,0 +1,269 @@
+import { SeoCrawlFetcher, SeoPageRenderer } from './types.js';
+import { SeoPageReading, SeoReadPageOptions } from './reading.js';
+import './facts.js';
+import './host-gate.js';
+
+/**
+ * THE AUDIT REPORT: what a standalone crawl ({@link crawlSite} in `./site`) hands back, as plain JSON
+ * a program can store, diff or turn into a file. Versioned by {@link AUDIT_REPORT_SCHEMA_VERSION}: a
+ * change that breaks a reader raises it; a new optional field does not.
+ *
+ * Every finding is a STABLE rule id from the catalog (`./catalog`) with its evidence as structured
+ * params, never English, so a reader can word it in any language and match on the id.
+ */
+/** The report format's version. */
+declare const AUDIT_REPORT_SCHEMA_VERSION = 1;
+/** How serious an issue is: `critical` (shown as an error), `warning` or `notice`. */
+type AuditReportSeverity = "critical" | "warning" | "notice";
+/**
+ * Whether a finding is a problem to fix (`issue`, counted and scored) or a small improvement that is
+ * not a problem (`tip`, listed apart, never counted as an issue).
+ */
+type AuditReportKind = "issue" | "tip";
+/** One page the crawl read. */
+interface AuditReportPage {
+    /** The page's URL, `https://` plus its key. */
+    url: string;
+    /** The page's key: host, path and query, normalized (`example.com/blog`). */
+    pageKey: string;
+    /** The status it answered (a redirect's own status), 0 when nothing answered, null when not read. */
+    status: number | null;
+    /** Why nothing usable answered (`timeout`, `network`, `status_403`...), or null. */
+    error: string | null;
+    /** Its title, or null when it has none or is not a page. */
+    title: string | null;
+    /** Clicks from the home page along followed links, or null when no such path reaches it. */
+    clickDepth: number | null;
+    /** Words in its main text, or null when it is not a page. */
+    wordCount: number | null;
+    /** How many issues (not tips) were filed on it. */
+    issues: number;
+}
+/** One finding on one page (a whole-site finding is filed on the home page). */
+interface AuditReportIssue {
+    /** The check's stable id (`title_missing`, `broken_internal_links`...). */
+    ruleId: string;
+    severity: AuditReportSeverity;
+    /** The area of the site the check is about (`metadata`, `links`, `performance`...). */
+    category: string;
+    kind: AuditReportKind;
+    /** Whether the check judges the whole site rather than one page. */
+    siteWide: boolean;
+    /** The page's URL. */
+    page: string;
+    /** The page's key. */
+    pageKey: string;
+    /** What the check found, as structured params (counts, examples, measured values). */
+    evidence: Record<string, unknown>;
+}
+/** The counts of a report. */
+interface AuditReportSummary {
+    /** Pages the crawl read. */
+    pages: number;
+    /** Issues filed (tips left out). */
+    issues: number;
+    /** Tips filed. */
+    tips: number;
+    /** Issues by severity. */
+    bySeverity: Record<AuditReportSeverity, number>;
+    /** Findings (issues and tips) by rule id. */
+    byRule: Record<string, number>;
+    /** Pages with at least one critical issue. */
+    pagesWithErrors: number;
+    /** The site's health score, 0 to 100, or null when no page could be judged. */
+    score: number | null;
+}
+/** What a standalone crawl found. */
+interface AuditReport {
+    schemaVersion: typeof AUDIT_REPORT_SCHEMA_VERSION;
+    /** The URL the crawl started from. */
+    startUrl: string;
+    /** When the crawl started and finished (ISO). */
+    startedAt: string;
+    finishedAt: string;
+    /**
+     * Whether the crawl read every page it found: false when it stopped at its page cap, so the checks
+     * that judge what is MISSING from the site (an orphan page, a page left out of the sitemap) held
+     * back.
+     */
+    complete: boolean;
+    /** Every page read, in the order it was found. */
+    pages: AuditReportPage[];
+    /** Every finding, issues before tips, the most serious first. */
+    issues: AuditReportIssue[];
+    summary: AuditReportSummary;
+}
+/**
+ * A report's findings as CSV (RFC 4180, CRLF line ends): one row per finding with its page, rule id,
+ * severity, category, kind, whether it is site-wide, and its evidence as JSON.
+ *
+ * @param report - The report.
+ * @returns The CSV text, the header first.
+ */
+declare function toCsv(report: AuditReport): string;
+/**
+ * A report as one standalone HTML page (no script, no external file): the counts, every finding
+ * grouped by rule (issues first, the most serious first), and every page read.
+ *
+ * @param report - The report.
+ * @returns The HTML document.
+ */
+declare function renderHtmlReport(report: AuditReport): string;
+
+/**
+ * THE CRAWLER'S ADDRESS GUARD: the transport a standalone crawl reads the web through by default, so a
+ * page that links or redirects to a private address (the machine itself, the local network, a cloud's
+ * metadata service) can never make the crawler fetch it.
+ *
+ * Every request, each redirect hop included (the crawler follows redirects itself, one request per
+ * hop), is checked before it is sent: http or https only, no credentials in the URL, the host resolved
+ * and refused when ANY address it resolves to is loopback, unspecified, private (RFC 1918), shared
+ * (CGNAT), link-local (the metadata address included), unique-local, multicast, reserved, or an IPv6
+ * address carrying such an IPv4 (mapped, NAT64, 6to4). The connection is then pinned to the very
+ * addresses checked, so a host that answers DNS with a public address and then a private one cannot
+ * be dialled on an address the guard never saw. A redirect is returned as it is, never followed.
+ *
+ * A refused request throws an error named `BlockedUrlError` whose `reason` says why (`refused`,
+ * `not_found` when the host does not exist, `dns_unavailable` when DNS could not answer), which the
+ * crawler files as a blocked, missing or unreachable URL.
+ */
+/** Why a URL failed the guard. */
+type BlockedUrlReason = "refused" | "not_found" | "dns_unavailable";
+/** Thrown when a URL fails the guard. */
+declare class BlockedUrlError extends Error {
+    readonly reason: BlockedUrlReason;
+    /**
+     * @param message - What was refused.
+     * @param reason - Why.
+     */
+    constructor(message: string, reason?: BlockedUrlReason);
+}
+/** How the guarded transport is built. */
+interface GuardedFetchOptions {
+    /**
+     * The host names (lower-cased; an IPv6 literal with or without its brackets) whose private or local
+     * addresses may be reached: a staging site on the local network or on this machine, named one by
+     * one. Every other host stays guarded, so a link from that site to another private address (a
+     * cloud's metadata service, the local network) is still refused. Empty by default.
+     */
+    allowPrivateHosts?: ReadonlySet<string>;
+}
+/**
+ * Whether an IP address is one the guard refuses to connect to.
+ *
+ * @param address - An IPv4 or IPv6 address.
+ * @returns Whether it is refused; anything that is not an IP address is refused.
+ */
+declare function isBlockedAddress(address: string): boolean;
+/**
+ * The guarded crawler transport ({@link guardedFetch}), the private addresses of the hosts named in
+ * `allowPrivateHosts` reachable. There is no unguarded mode: every request is still checked, looked up
+ * and pinned.
+ *
+ * @param options - The hosts whose private addresses may be reached.
+ * @returns The transport.
+ */
+declare function createGuardedFetch(options?: GuardedFetchOptions): SeoCrawlFetcher;
+/**
+ * THE DEFAULT TRANSPORT of a standalone crawl (`crawlSite`) and of `readPage`: every request checked against
+ * private, local and reserved addresses, its connection pinned to the addresses checked, its redirect
+ * returned rather than followed, and cut at its timeout.
+ */
+declare const guardedFetch: SeoCrawlFetcher;
+
+/**
+ * THE STANDALONE CRAWL: a whole site crawled in this process with the same reading, discovery and
+ * rules the SEOStudio app uses, held in memory instead of a database, and judged by every rule that
+ * needs no paid data. And one page's reading, for a client that fetches from its own address.
+ *
+ * What it leaves out, because it needs an account or the app's history: Search Console, Analytics,
+ * lab tests and CrUX data (their rules stay quiet), the TLS read of the site's host, the members'
+ * custom checks and AI prompts, and a page's change history. A one-off crawl confirms an outbound
+ * link as gone by asking it twice; a server error on another site is never confirmed within one run
+ * (an outage passes).
+ */
+/** Where a standalone crawl stands, as {@link CrawlSiteOptions.onProgress} is told. */
+interface CrawlSiteProgress {
+    /** `plan`: robots.txt and the sitemaps; `pages`: the pages; `resources`: what they load; `audit`: the rules. */
+    phase: "plan" | "pages" | "resources" | "audit" | "done";
+    /** Pages read so far. */
+    pagesRead: number;
+    /** Pages found so far, read or not (never more than the page cap). */
+    pagesFound: number;
+    /** The URL being read, or null between two reads. */
+    currentUrl: string | null;
+}
+/** How a standalone crawl runs. */
+interface CrawlSiteOptions {
+    /** The site's start URL, e.g. `https://example.com/`. */
+    url: string;
+    /** The most pages known to the crawl, read or not; 500 by default. */
+    maxPages?: number;
+    /** Pages read at once; 2 by default. Each host is still asked at its polite pace. */
+    concurrency?: number;
+    /** The User-Agent every request sends; the crawler's own by default. */
+    userAgent?: string;
+    /** Told whenever the crawl moves on. */
+    onProgress?: (progress: CrawlSiteProgress) => void;
+    /**
+     * The host names (lower-cased) whose private or local addresses the crawl may reach: a staging site
+     * on the local network or on this machine, named one by one. Every other request, each redirect hop
+     * included, is refused when its host resolves to a private, local or reserved address
+     * (`guardedFetch`), so a link from the staging site to another private address is still refused.
+     */
+    allowPrivateHosts?: ReadonlySet<string>;
+    /**
+     * The transport every request goes through, in place of the address guard (redirects are followed
+     * hop by hop, so it must not follow them itself). Tests pass a fake site.
+     */
+    fetch?: SeoCrawlFetcher;
+    /** A browser to render each page in; without one every page is judged on its raw HTML. */
+    render?: SeoPageRenderer;
+    /** The clock; the wall clock by default. */
+    now?: () => Date;
+    /** The pause between requests; a real one by default. */
+    sleep?: (ms: number) => Promise<void>;
+}
+/** One page's reading, the shape a server accepts from a client that fetched the page itself. */
+type PageReading = SeoPageReading;
+/** How one page is read. */
+interface ReadPageOptions extends SeoReadPageOptions {
+    /** The User-Agent the request sends; the crawler's own by default. */
+    userAgent?: string;
+    /** The host names (lower-cased) whose private or local addresses may be reached; none by default. */
+    allowPrivateHosts?: ReadonlySet<string>;
+    /** The transport, in place of the address guard (redirects followed hop by hop). */
+    fetch?: SeoCrawlFetcher;
+    /** A browser to render the page in; without one the reading is raw only. */
+    render?: SeoPageRenderer;
+    /** The clock; the wall clock by default. */
+    now?: () => Date;
+    /** The pause before a retried request; a real one by default. */
+    sleep?: (ms: number) => Promise<void>;
+}
+/**
+ * READS ONE PAGE as the SEOStudio desktop app does: the fetch (redirects followed hop by hop, a server
+ * error asked again once), the extractor's read of its HTML, and the rendered read when a browser is
+ * lent. The reading is cut to the bounds a server accepts, so a client can send it as it is.
+ *
+ * @param url - The page's URL.
+ * @param options - How it is read.
+ * @returns The reading.
+ * @throws When the URL names no http(s) page.
+ */
+declare function readPage(url: string, options?: ReadPageOptions): Promise<PageReading>;
+/**
+ * CRAWLS A SITE in this process and audits it: robots.txt and the sitemaps, then every page found from
+ * the home page and the sitemaps (up to the cap), what the pages load and link out to, the site's host
+ * variants, and every rule that needs no paid data. Polite as the app's crawler: robots.txt obeyed,
+ * its `Crawl-delay` kept (one second at least between two requests to a host), a 429 or 503 retried
+ * only after a pause.
+ *
+ * @param options - The site, the caps and the progress hook.
+ * @returns The report.
+ * @throws When the URL names no http(s) site.
+ */
+declare function crawlSite(options: CrawlSiteOptions): Promise<AuditReport>;
+
+export { AUDIT_REPORT_SCHEMA_VERSION, BlockedUrlError, crawlSite, createGuardedFetch, guardedFetch, isBlockedAddress, readPage, renderHtmlReport, toCsv };
+export type { AuditReport, AuditReportIssue, AuditReportKind, AuditReportPage, AuditReportSeverity, AuditReportSummary, BlockedUrlReason, CrawlSiteOptions, CrawlSiteProgress, GuardedFetchOptions, PageReading, ReadPageOptions };