@seostudio/audit-engine 0.0.0-stage → 0.1.0

package/dist/reading.d.ts

@@ -0,0 +1,517 @@
+import { SeoPageRenderOutcomeValue, SeoPageRenderResourceFailure, SeoPageDomFacts, SeoPageA11yFacts, SeoPageDesktopView, SeoPageCheckFacts, SeoCustomCheck, SeoRobotsFetchOutcomeValue, SeoSoft404Probe, SeoSitemapFileFacts, SeoAgentFilesFacts, SeoRobotsFetchFacts } from './facts.js';
+import { SeoHostGate } from './host-gate.js';
+import { FetchedResource, ExtractedPage, SeoRenderScript, SeoCrawlFetcher, SeoPageRenderer, SeoBrowserRender } from './types.js';
+
+/**
+ * The bounds a reading carried to the server keeps. Far above any real page: they exist so one upload
+ * can never be unbounded, and a page past them is read the same way minus the overflow. In a file of
+ * its own so the route's validator loads none of the extractor.
+ */
+declare const SEO_READING_LIMITS: {
+    /** Characters of a page's main text. */
+    readonly mainTextChars: 2000000;
+    /** Links of one page. */
+    readonly links: 10000;
+    /** Characters of a URL, an anchor or a header value. */
+    readonly textChars: 4000;
+    /** Entries of a list the audit only counts or samples (mixed content, unsized images, canonicals). */
+    readonly listItems: 500;
+    /** JSON-LD blocks, and their characters together. */
+    readonly jsonLdBlocks: 200;
+    readonly jsonLdChars: 1000000;
+    /** hreflang alternates. */
+    readonly hreflang: 1000;
+    /** Schema types. */
+    readonly schemaTypes: 200;
+    /** Hops of a redirect chain. */
+    readonly chainHops: 20;
+    /** Readings in one batch. */
+    readonly batchPages: 50;
+    /** Sitemap URLs in a plan reading, the characters of each, and sitemap files. */
+    readonly sitemapUrls: number;
+    readonly sitemapUrlChars: 2048;
+    readonly sitemapFiles: number;
+    /** Characters of a robots.txt. */
+    readonly robotsChars: number;
+    /** Console lines of a render, the characters of each, and the failed resources it names. */
+    readonly renderConsoleLines: 20;
+    readonly renderConsoleChars: 300;
+    readonly renderFailures: 50;
+    /** Resource check readings in one upload: one hand-out's worth. */
+    readonly resourceReadings: number;
+};
+
+/**
+ * THE GRAMMAR CHECK OF A PAGE, on a member's computer only: the host lends a checker (the desktop app's
+ * Harper, an English grammar checker compiled to WebAssembly that runs on the computer, no server and no
+ * Java), and this file decides what it reads and what is kept. Only English pages are read (a page
+ * whose language says English, or says none), the title, the H1 and the main text, bounded; only the
+ * problem's kind, its text and the first suggestion are kept, never an English message.
+ *
+ * PRECISION FIRST: only the kinds that name a mistake in a sentence are kept ({@link GRAMMAR_KINDS}),
+ * and never one that spans two lines (two table cells or list items read as one sentence).
+ * The checker's other kinds misfire on web text, where headings, menus, list items, table cells and code
+ * are joined into one text: a heading or a list item reads as a sentence without a capital or a full
+ * stop, every straight quote as unbalanced, a table as one 150-word sentence, and style advice ("config"
+ * to "configuration", an Oxford comma) is not a mistake. Spelling is left to the spelling check, which
+ * judges a word against the site's own vocabulary. Pure apart from the checker.
+ */
+/** One problem the checker found. */
+interface SeoGrammarLint {
+    /** The kind of problem, as the checker names it (`Grammar`, `Repetition`, ...). */
+    kind: string;
+    /** The text it is about. */
+    text: string;
+    /** The first replacement it suggests, or null. */
+    suggestion: string | null;
+}
+/** Checks a text's grammar; the host's checker (Harper on the desktop app). Rejects when it cannot. */
+type SeoGrammarChecker = (text: string) => Promise<SeoGrammarLint[]>;
+
+/** One rule, split once for matching. */
+interface CompiledRule {
+    allow: boolean;
+    /** The rule's length, which decides the longest match. */
+    length: number;
+    /** The rule split on `*`. */
+    parts: string[];
+    /** Whether the rule ends in `$`. */
+    anchored: boolean;
+    /** The rule as the file writes it, e.g. `Disallow: /private`. */
+    line: string;
+    /** The `User-agent` name of the group it came from (`*` for the catch-all group). */
+    userAgent: string;
+}
+/** The rules that govern one agent, ready to match. */
+interface CompiledRobots {
+    rules: CompiledRule[];
+    /** The pause the file asks between requests, in ms, or null. */
+    crawlDelayMs: number | null;
+    sitemaps: string[];
+}
+
+/** Which languages a reading checks: those light enough for the server, or every one (a member's computer). */
+type SeoSpellingScope = "server" | "all";
+
+/**
+ * WHAT THE CRAWL SENDS A SITE THAT ASKS TO SIGN IN: the members' extra request headers, HTTP basic
+ * auth and (on the desktop app) the cookies of a recorded login, as one set of headers - sent ONLY to
+ * the project's own host and its `www.` twin, never to another site a page links to or loads from,
+ * and checked again at every redirect hop, so a redirect off the site drops them.
+ *
+ * PURE: no database, no network; a desktop main process can run it.
+ */
+/** The headers a request to one URL carries for the site, or null when the URL is not on the site. */
+type SeoSiteHeaders = (url: string) => Record<string, string> | null;
+
+/** What a sitemap's hreflang alternates (`<xhtml:link rel="alternate" hreflang>`) showed. */
+interface SitemapHreflang {
+    /** Page entries that carry alternates. */
+    urls: number;
+    /** Whether an alternate was written without the XHTML namespace declared for it. */
+    namespaceMissing: boolean;
+    /** Entries whose set leaves out the entry itself, or differs from the set a listed alternate carries. */
+    incomplete: number;
+    /** A few of those entries' URLs. */
+    examples: string[];
+}
+/** What one `<url>` entry of a sitemap states beyond its URL. */
+interface ParsedSitemapEntry {
+    /** The `<lastmod>` as written, or null. */
+    lastmod: string | null;
+    /** Whether it carries a tag Google ignores (`priority`, `changefreq`, a retired image tag). */
+    deprecated: boolean;
+    /** How many `<image:image>` it lists. */
+    images: number;
+    /** How many `<video:video>` lack a required tag (a thumbnail, a title, a description, a content or player URL). */
+    videosIncomplete: number;
+    /** Each `<news:publication_date>` as written. */
+    newsDates: string[];
+}
+/** One parsed sitemap file. */
+interface ParsedSitemap {
+    /** Page URLs (`<url><loc>`), the first {@link SITEMAP_KEPT_URLS}. */
+    urls: string[];
+    /** Every page URL the file lists, kept or not. */
+    listed: number;
+    /** Each page URL's `<lastmod>` day (`YYYY-MM-DD`), or null when it has none; aligned with `urls`. */
+    lastmods: (string | null)[];
+    /** Each page URL's extra facts; aligned with `urls`. */
+    entries: ParsedSitemapEntry[];
+    /** Nested sitemaps (`<sitemap><loc>`). */
+    sitemaps: string[];
+    /**
+     * The root element's local name (`urlset`, `sitemapindex`, or a feed's `rss` or `feed`), `text`
+     * for a plain list of URLs, or null when it is none of them.
+     */
+    root: string | null;
+    /** Whether the root declares a sitemap namespace (a feed or a text list needs none). */
+    namespaced: boolean;
+    /** Whether the XML is well formed: every element closed, in order (a cut or broken file is not). */
+    wellFormed: boolean;
+    /** The hreflang alternates its entries carry, or undefined when none does. */
+    hreflang?: SitemapHreflang;
+}
+
+/** The `Accept` header of a page fetch. */
+declare const SEO_ACCEPT_HTML = "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5";
+/** Statuses that say "not now" rather than anything about the page. */
+declare const SEO_FAILURE_STATUSES: ReadonlySet<number>;
+/** What a reading reaches outside itself: the guarded transport, the clock, a pause and the browser. */
+interface SeoReadDeps {
+    fetch: SeoCrawlFetcher;
+    now: () => Date;
+    sleep: (ms: number) => Promise<void>;
+    /** The browser a page is rendered in; a reading made without one is raw only. */
+    render?: SeoPageRenderer;
+    /** The pace every crawl of a host shares (the server's); without one, a reading only pauses itself. */
+    hostGate?: SeoHostGate;
+    /**
+     * The site's sign-in headers (`./site-access`), for the requests the rendering browser makes to the
+     * site; `fetch` already adds them to the crawl's own requests (`withSiteAccess`).
+     */
+    siteHeaders?: SeoSiteHeaders;
+}
+/**
+ * Waits before a request to a site: for the host's shared slot when a gate is at hand (every crawl of
+ * the host takes turns), else the plain pause.
+ *
+ * @param deps - The clock, the pause and the gate.
+ * @param url - The URL about to be asked for.
+ * @param delayMs - The least time between two requests to the host.
+ * @param first - Whether this is the reading's first request (no plain pause before it).
+ */
+declare function paceRequest(deps: Pick<SeoReadDeps, "now" | "sleep" | "hostGate">, url: string, delayMs: number, first?: boolean): Promise<void>;
+/**
+ * The pause between two requests to the site: the robots.txt `Crawl-delay`, never below the configured
+ * floor nor above its ceiling.
+ *
+ * @param robots - The site's robots.txt.
+ * @returns The pause in ms.
+ */
+declare function politeDelay(robots: CompiledRobots): number;
+/**
+ * A robots.txt compiled for the group the crawl obeys: the crawler's own token by default.
+ *
+ * @param text - The file, or null when none was read.
+ * @param agent - The product token whose group is obeyed (`robotsAgentOf` in `./discovery`).
+ * @returns The compiled rules.
+ */
+declare function crawlerRobots(text: string | null, agent?: string): CompiledRobots;
+/** What a plan reading needs to know about the site, from the server. */
+interface SeoSitePlanInput {
+    /** The project's website. */
+    siteUrl: string;
+    /** The User-Agent every request sends. */
+    userAgent: string;
+    /** Whether the project holds a verified property for its site (every site is crawled whole). */
+    verified: boolean;
+    /** The robots.txt group the crawl obeys; the crawler's own when absent (an older server). */
+    robotsAgent?: string;
+    /** Whether the soft-404 probe is due. */
+    probeSoft404: boolean;
+    /** The robots.txt read last time, used when the site does not answer for it now. */
+    fallbackRobotsTxt: string | null;
+    /** The last soft-404 probe, for a plan that does not probe again; absent from an older server. */
+    soft404?: SeoSoft404Probe | null;
+}
+/** What the plan read from the site. */
+interface SeoSitePlanReading {
+    /**
+     * The robots.txt the crawl obeys: the file the site served, empty when it answered 4xx (no file
+     * allows everything), the last one read when it did not answer, or null when there is none at all.
+     */
+    robotsTxt: string | null;
+    sitemap: {
+        /** Whether any sitemap could be read. */
+        read: boolean;
+        /** Each listed page on the site's own hosts, its URL as written (one per page key). */
+        urls: string[];
+        /** Each url set file's facts. */
+        files: SeoSitemapFileFacts[];
+    };
+    /** The probe for a page that cannot exist, or null when it was not made (or nothing usable answered). */
+    soft404: SeoSoft404Probe | null;
+    /**
+     * How robots.txt answered; absent on readings from an app made before it was read. Stored only from
+     * the server's own read: the server replaces whatever a member's computer sends with it.
+     */
+    robotsFetch?: SeoRobotsFetch | null;
+    /** The site's files for AI agents; absent on readings from an app made before they were read. */
+    agentFiles?: SeoAgentFilesFacts | null;
+}
+/**
+ * What one url set file states for the sitemap file rules: its URLs out of scope, its invalid and
+ * future dates, the tags Google ignores, the extensions' limits, and URLs an earlier file listed.
+ *
+ * @param parsed - The parsed file.
+ * @param fileUrl - Where the file answered.
+ * @param declared - Whether robots.txt named the file (or the index that named it): it may then list
+ *   any URL of the site, not only its own folder's.
+ * @param onSite - Whether a host is one of the site's own.
+ * @param listed - The URLs earlier files listed; this file's are added.
+ * @param now - The read's time.
+ * @returns The facts beyond the counts.
+ */
+declare function sitemapFileFacts(parsed: ParsedSitemap, fileUrl: string, declared: boolean, onSite: (host: string) => boolean, listed: Set<string>, now: Date): Omit<SeoSitemapFileFacts, "url" | "urls" | "withLastmod" | "distinctLastmod" | "source" | "status">;
+/** How robots.txt answered one read. */
+interface SeoRobotsFetch {
+    /** The last answer's status, or null when nothing answered. */
+    status: number | null;
+    outcome: SeoRobotsFetchOutcomeValue;
+}
+/** One read of robots.txt: the file the crawl obeys, and how the site answered for it. */
+interface SeoRobotsRead {
+    /** The file, "" for a 4xx, the last one read when the site did not answer, or null when none. */
+    text: string | null;
+    /** How the site answered, or null when the website is not a crawlable URL. */
+    fetch: SeoRobotsFetch | null;
+}
+/**
+ * Reads the robots.txt a crawl obeys from the site's own origin: the file it served, empty when it
+ * answered 4xx (no file allows everything), the last one read when it did not answer, or null when
+ * there is none at all; beside it, how the site answered.
+ *
+ * @param input - The site, the User-Agent and the last robots.txt read.
+ * @param deps - The transport and the clock.
+ * @returns The robots.txt and how the site answered.
+ */
+declare function readSiteRobots(input: Pick<SeoSitePlanInput, "siteUrl" | "userAgent" | "fallbackRobotsTxt">, deps: Pick<SeoReadDeps, "fetch" | "now"> & Partial<Pick<SeoReadDeps, "sleep" | "hostGate">>): Promise<SeoRobotsRead>;
+/**
+ * How robots.txt answered at a plan, with the run of plans in a row it was unreachable at.
+ *
+ * @param fetch - How it answered now.
+ * @param before - The last plan's facts, or null.
+ * @param now - Now.
+ * @returns The facts to store.
+ */
+declare function robotsFetchFacts(fetch: SeoRobotsFetch, before: SeoRobotsFetchFacts | null, now: Date): SeoRobotsFetchFacts;
+/**
+ * READS WHAT A CRAWL PLAN NEEDS FROM THE SITE: robots.txt, every sitemap file and, when it is due, the
+ * soft-404 probe. When there is no robots.txt to obey at all, nothing else is fetched.
+ *
+ * @param input - The site and what the server decided about it.
+ * @param deps - The transport, the clock and the pause.
+ * @returns The reading, or null when the website is not a crawlable URL.
+ */
+declare function readSitePlan(input: SeoSitePlanInput, deps: SeoReadDeps): Promise<SeoSitePlanReading | null>;
+/** What one fetch answered, as a reading carries it: the body left behind once it was read. */
+type SeoReadResource = Omit<FetchedResource, "body" | "requestedUrl" | "truncated">;
+/** The page as a browser rendered it, read by the same extractor as its raw HTML. */
+interface SeoRenderedReading {
+    outcome: SeoPageRenderOutcomeValue;
+    /** Why the render failed, or null. */
+    error: string | null;
+    /** The URL of the document the browser ended on. */
+    finalUrl: string;
+    renderMs: number;
+    /** The extractor's read of the rendered DOM; null when the render failed. */
+    page: ExtractedPage | null;
+    /** Uncaught script errors and console errors, cut short. */
+    consoleErrors: string[];
+    /** Scripts, stylesheets and data requests the page could not load. */
+    failedResources: SeoPageRenderResourceFailure[];
+    /** Requests the render refused to send (a POST to another host, a private address); absent from an older app. */
+    blockedRequests?: number;
+    /** Requests a host refused for the crawl's rate limit; absent from an older app. */
+    rateLimitedRequests?: number;
+    /** What the in-page probe measured; absent from an app from before it ran, null when it could not. */
+    dom?: SeoPageDomFacts | null;
+    /** What the accessibility checks found; absent or null when they did not run. */
+    a11y?: SeoPageA11yFacts | null;
+    /** The page as a desktop browser rendered it (the desktop app's second render), when there was one. */
+    desktop?: SeoPageDesktopView | null;
+}
+/** One page's reading: what answered, the extractor's read of its raw HTML, and of its render. */
+interface SeoPageReading {
+    /** The page fetched. */
+    pageKey: string;
+    resource: SeoReadResource;
+    /**
+     * The page read from the RAW answer: only for a 2xx HTML answer that did not redirect to another
+     * page, null otherwise (a redirect, an error status, a failure, a file that is not HTML).
+     */
+    page: ExtractedPage | null;
+    /**
+     * The page as a browser rendered it, whenever the raw answer was a page; null when it was not (or
+     * the reading was made without a browser, by an older desktop app).
+     */
+    rendered: SeoRenderedReading | null;
+    /** What the optional checks found on the page (spelling, AMP, custom patterns); absent or null when none ran. */
+    checks?: SeoPageCheckFacts | null;
+    /**
+     * A JPEG of the rendered page (base64), only when the reading asked for one (a single page
+     * analysis, or a crawl on a member's computer that keeps screenshots); never carried to the server
+     * with a crawl's readings.
+     */
+    screenshot?: string | null;
+    /**
+     * The page's raw and rendered HTML, only when the reading asked for them (a crawl on a member's
+     * computer that keeps them for itself); never carried to the server.
+     */
+    html?: {
+        raw: string;
+        rendered: string | null;
+    } | null;
+}
+/** Where a fetch answered from, as the crawl judges it. */
+interface SeoReadVerdict {
+    /** The page the fetch landed on, when it is a crawlable URL. */
+    finalKey: string | null;
+    /** Whether the chain moved to another page. */
+    redirected: boolean;
+    /** The status the page is filed under: a redirecting page's own redirect status, else the last hop's. */
+    status: number;
+    /** Whether nothing usable answered (a transport failure, nothing at all). */
+    failed: boolean;
+}
+/**
+ * How the crawl reads a fetch: failed or not, redirected or not, and the status it files.
+ *
+ * @param pageKey - The page fetched.
+ * @param resource - What answered.
+ * @returns The verdict.
+ */
+declare function readVerdict(pageKey: string, resource: SeoReadResource): SeoReadVerdict;
+/**
+ * Whether a reading is about the page it names: its first hop (or, with no hop, the URL it failed on)
+ * is that page, and a fetch that answered ends on the last hop of its own chain. A reading made
+ * somewhere else carries its URLs, so a reading that names one page and describes another is refused.
+ *
+ * @param pageKey - The page the reading names.
+ * @param resource - What answered.
+ * @returns Whether the reading describes that page.
+ */
+declare function readingFitsPage(pageKey: string, resource: SeoReadResource): boolean;
+/**
+ * Whether a fetch's answer is a page the extractor reads: a 2xx HTML answer that stayed on the page.
+ *
+ * @param pageKey - The page fetched.
+ * @param resource - What answered.
+ * @returns Whether its body is read as a page.
+ */
+declare function readsAsPage(pageKey: string, resource: SeoReadResource): boolean;
+/**
+ * Whether a 2xx answer that stayed on the page holds an HTML document its `Content-Type` does not
+ * name as HTML (missing, `text/plain`, ...): browsers may still show it, but search engines read it
+ * by its type.
+ *
+ * @param pageKey - The page fetched.
+ * @param resource - What answered.
+ * @param body - Its body, or null.
+ * @returns Whether it does.
+ */
+declare function htmlUnlabelled(pageKey: string, resource: SeoReadResource, body: Uint8Array | null): boolean;
+/**
+ * WHAT GOOGLEBOT'S CUT LEAVES OUT of an HTML file over what it reads ({@link GOOGLEBOT_FILE_BYTES}):
+ * the page read again from only that first part, against the whole read. The head is cut when the
+ * part reads a head element search engines use otherwise (the title, the description, the robots meta,
+ * the canonical, hreflang); the main text is cut when the part holds fewer than
+ * {@link CUT_MAIN_TEXT_KEPT} of its words.
+ *
+ * @param html - The page's HTML as served, decoded.
+ * @param page - The whole read.
+ * @param finalUrl - The URL it was read at.
+ * @returns What the cut leaves out, or null for a file within the limit.
+ */
+declare function htmlCutOf(html: string, page: ExtractedPage, finalUrl: string): {
+    head: boolean;
+    mainText: boolean;
+} | null;
+/**
+ * What the extractor reads from a browser's render, judged against the page it was asked for: a render
+ * that ended on another document (a script navigated away) or on a bot wall reads as failed.
+ *
+ * @param pageKey - The page.
+ * @param render - What the browser answered.
+ * @returns The rendered reading.
+ */
+declare function renderedReadingOf(pageKey: string, render: SeoBrowserRender): SeoRenderedReading;
+/**
+ * Whether a subresource may be fetched by the rendering browser: anything off the site, and on the
+ * site's own hosts what its robots.txt lets the crawler fetch (Google obeys robots.txt for a page's
+ * scripts, styles and data too).
+ *
+ * @param pageUrl - The page rendered.
+ * @param robots - The site's robots.txt, or null when none is known (everything allowed).
+ * @returns The check, or null when nothing is checked.
+ */
+declare function renderResourceCheck(pageUrl: string, robots: CompiledRobots | null): ((url: string) => boolean) | null;
+/** How a page reading is made beyond its transport. */
+interface SeoReadPageOptions {
+    /** The robots.txt the crawl obeys, for the rendering browser's own requests to the site. */
+    robots?: CompiledRobots | null;
+    /** Told just before the page is rendered. */
+    onRender?: () => void;
+    /** Which accessibility rules run on the rendered page: the search-relevant subset, all, or none. */
+    accessibility?: "seo" | "all" | null;
+    /** Whether the page is also rendered as a desktop browser sees it, for the parity check. */
+    desktopRender?: boolean;
+    /** Which languages the spelling check reads (the server's light ones, or all), or null for none. */
+    spelling?: SeoSpellingScope | null;
+    /** The project's custom search and extraction patterns. */
+    customChecks?: readonly SeoCustomCheck[];
+    /**
+     * A second robots.txt the browser also obeys for the site's own files (a desktop run's test file):
+     * a file either one forbids is not loaded.
+     */
+    extraRobots?: CompiledRobots | null;
+    /** Whether the rendered page is also captured as a screenshot (a single page analysis). */
+    screenshot?: boolean;
+    /** The device the page is rendered as: a phone (the default, as Google renders) or a desktop. */
+    device?: "mobile" | "desktop";
+    /** How long a rendered page is left to run once it settled, in ms (the members' render wait). */
+    renderWaitMs?: number;
+    /** The project's custom JavaScript snippets, run on the rendered page (a member's computer only). */
+    scripts?: readonly SeoRenderScript[];
+    /** The grammar checker the host lends (a member's computer only), or null for no grammar check. */
+    grammar?: SeoGrammarChecker | null;
+    /**
+     * Whether the reading keeps the page's raw and rendered HTML (`html`), for the member's computer to
+     * keep for itself; never carried to the server.
+     */
+    keepHtml?: boolean;
+}
+/**
+ * A desktop render's view of the page: what the parity check compares with the phone render.
+ *
+ * @param pageKey - The page.
+ * @param render - What the desktop browser answered.
+ * @returns The view.
+ */
+declare function desktopViewOf(pageKey: string, render: SeoBrowserRender): SeoPageDesktopView;
+/**
+ * READS ONE PAGE: one fetch through the guarded transport (redirects followed hop by hop, the body
+ * capped; a server error asked for once more after the polite pause, {@link RETRIED_SERVER_ERRORS};
+ * a platform's mobile copy read as the page itself, {@link readMobileCopyAsPage}),
+ * the extractor's read of it when it is a page, and - when a browser is at hand - the same
+ * extractor's read of the page as the browser rendered that very HTML. With a browser at hand the page
+ * is asked for with the browser's own User-Agent (`renderedDocumentUserAgent`).
+ *
+ * @param pageKey - The page (a normalized key; fetched over https).
+ * @param userAgent - The User-Agent.
+ * @param deps - The transport, the clock, the browser and (on the server) the per-host pace its
+ *   requests to the site's own host share with every crawl of it.
+ * @param options - The robots.txt the browser obeys and a hook before it renders.
+ * @returns The reading.
+ */
+declare function readSitePage(pageKey: string, userAgent: string, deps: Pick<SeoReadDeps, "fetch" | "now" | "render"> & Partial<Pick<SeoReadDeps, "sleep" | "hostGate" | "siteHeaders">>, options?: SeoReadPageOptions): Promise<SeoPageReading>;
+/**
+ * A plan reading cut to {@link SEO_READING_LIMITS}: a sitemap URL too long to be a page is left out,
+ * as the server's own crawl would never fetch it.
+ *
+ * @param reading - The plan reading.
+ * @returns The bounded reading.
+ */
+declare function boundPlanReading(reading: SeoSitePlanReading): SeoSitePlanReading;
+/**
+ * A reading cut to {@link SEO_READING_LIMITS}, so it can be carried to the server whole. A page
+ * within the limits (every real page) comes back unchanged.
+ *
+ * @param reading - The reading.
+ * @returns The bounded reading.
+ */
+declare function boundReading(reading: SeoPageReading): SeoPageReading;
+
+export { SEO_ACCEPT_HTML, SEO_FAILURE_STATUSES, SEO_READING_LIMITS, boundPlanReading, boundReading, crawlerRobots, desktopViewOf, htmlCutOf, htmlUnlabelled, paceRequest, politeDelay, readSitePage, readSitePlan, readSiteRobots, readVerdict, readingFitsPage, readsAsPage, renderResourceCheck, renderedReadingOf, robotsFetchFacts, sitemapFileFacts };
+export type { SeoPageReading, SeoReadDeps, SeoReadPageOptions, SeoReadResource, SeoReadVerdict, SeoRenderedReading, SeoRobotsFetch, SeoRobotsRead, SeoSitePlanInput, SeoSitePlanReading };

package/dist/reading.js

@@ -0,0 +1,3 @@
+export { SEO_ACCEPT_HTML, SEO_FAILURE_STATUSES, boundPlanReading, boundReading, crawlerRobots, desktopViewOf, htmlCutOf, htmlUnlabelled, paceRequest, politeDelay, readSitePage, readSitePlan, readSiteRobots, readVerdict, readingFitsPage, readsAsPage, renderResourceCheck, renderedReadingOf, robotsFetchFacts, sitemapFileFacts } from './chunk-XTLRDZOD.js';
+export { SEO_READING_LIMITS } from './chunk-7SGWGESF.js';
+import './chunk-6QPF6O7D.js';

package/dist/resource-check.d.ts

@@ -0,0 +1,131 @@
+import { SeoResourceKindValue, SeoResourceCheckMethodValue, SeoPageRedirectHop, SeoResourceHeaders, SeoResourceCheckErrorValue } from './facts.js';
+import { SeoHostGate } from './host-gate.js';
+import { SeoCrawlFetcher } from './types.js';
+
+/**
+ * THE RESOURCE CHECK'S NETWORK HALF: one URL a site's pages load or link out to, asked about through
+ * the crawler's guarded transport (`safeFetch` in production: every hop SSRF-checked again, redirects
+ * followed here by hand), and a list of them read politely: each host one request at a time with a pause
+ * between two requests to it, a few hosts at once. Plain JSON out and NO database, so the server's crawl job and the
+ * desktop app's main process run the very same check.
+ *
+ * A file a page loads gets a HEAD, then a ranged GET (`Range: bytes=0-0`) when the HEAD was refused or
+ * did not say how big the file is; a server that ignores the range has its body counted up to a cap and
+ * cut. An outbound link gets a HEAD alone, and a ranged GET only where the HEAD was refused or answered
+ * any failure but a 429 (some servers fail a HEAD they would answer by GET): its status is all its
+ * check needs. An image is asked for with a browser's `Accept` header (a server that sends WebP or
+ * AVIF to browsers answers as it does for them), and one of some weight gets one more ranged GET of its
+ * first bytes, where its header states its pixel size (an image much larger than the page shows it is
+ * wasted bytes) and a GIF shows whether it is animated. A text file a page
+ * loads (a script, a stylesheet, an SVG, JSON) whose answer named no compression is asked once more by
+ * a plain GET offering gzip and brotli, its body never read: many servers compress a GET but not a
+ * HEAD or a ranged answer. A compressed script or stylesheet of some weight is read once by a plain GET,
+ * up to just past what Googlebot reads of a file, for its unpacked size. A linked PDF gets two ranged
+ * GETs more, of its first and its last bytes, where its title is written. A link's spelling of a page of
+ * the site is asked once, its redirect never followed: what that one answer says is all its check needs.
+ */
+/** Images lighter than this are not asked for their pixel size: shown too large, they waste little. */
+declare const SEO_IMAGE_SIZE_MIN_BYTES: number;
+/** Text under this many bytes gains too little from compression to be judged. */
+declare const SEO_COMPRESS_MIN_BYTES: number;
+/**
+ * Whether a file is text worth compressing: a text content type, and for an image, an icon, a font or
+ * a media file, an SVG only (a binary file a server labels as text is not).
+ *
+ * @param kind - How a page uses it.
+ * @param contentType - The `Content-Type`, or null.
+ * @returns Whether it is.
+ */
+declare function seoCompressible(kind: SeoResourceKindValue, contentType: string | null): boolean;
+/** One URL to check. */
+interface SeoResourceCheckTarget {
+    url: string;
+    kind: SeoResourceKindValue;
+}
+/**
+ * What one check found. No response header is kept beyond the content type, the size and, for a file
+ * a page loads, its caching and compression headers (the rules read no other), and a failure is only
+ * ever one of the {@link SeoResourceCheckErrorValue} codes, never a transport's message.
+ */
+interface SeoResourceReading {
+    /** The URL checked, as it was handed out. */
+    url: string;
+    kind: SeoResourceKindValue;
+    /** The request whose answer is kept: a HEAD, or the GET that followed it. */
+    method: SeoResourceCheckMethodValue;
+    /** The first hop's status (a redirecting URL is a redirect), 0 when nothing answered. */
+    status: number;
+    /** The last hop's status, 0 when nothing answered. */
+    finalStatus: number;
+    finalUrl: string;
+    chain: SeoPageRedirectHop[];
+    contentType: string | null;
+    /** The body's size in bytes as served, or null when the server never said. */
+    sizeBytes: number | null;
+    /**
+     * An image's pixel size, from its file header; null when it was not read (not an image, a light
+     * one, an SVG, a format not known); absent from an app from before it was read.
+     */
+    width?: number | null;
+    height?: number | null;
+    /**
+     * The caching and compression headers of a file a page loads; null for an outbound link or when
+     * nothing answered; absent from an app from before they were kept (an older app's `headers`, read
+     * from a HEAD alone, are never taken for them).
+     */
+    delivery?: SeoResourceHeaders | null;
+    /**
+     * A linked document's title from its metadata (a PDF's): `''` when it holds none, null when it was
+     * not read; absent from an app from before it was read.
+     */
+    title?: string | null;
+    responseMs: number;
+    error: SeoResourceCheckErrorValue | null;
+}
+/** What a check reaches outside itself. */
+interface SeoResourceCheckDeps {
+    fetch: SeoCrawlFetcher;
+    now: () => Date;
+    sleep: (ms: number) => Promise<void>;
+    /** The per-host pace every crawl of a host shares (the server's); without one a list paces itself. */
+    hostGate?: SeoHostGate;
+}
+/** How a list of checks is read. */
+interface SeoReadResourcesOptions {
+    /** The pause between two requests to the same host, in ms (a shared CDN host's is shorter). */
+    delayMs: number;
+    /**
+     * The site's own host. When given, `delayMs` (which may hold the site's own `Crawl-delay`) paces
+     * that host alone; every other host gets the crawler's own pause, so a site's robots.txt never
+     * slows the shared pace of a host it does not own (an image CDN, another site).
+     */
+    siteHost?: string;
+    /** Asked before each check; true stops the list there (a deadline, a pause, a stop). */
+    stop?: () => boolean;
+    /** Told the URL about to be checked. */
+    onCheck?: (url: string) => void;
+}
+/**
+ * How many hosts a list of checks reads at once. Each host is still read one check at a time, at its
+ * own pace: a page's files spread over a few hosts (the site, its image and script CDNs, the tags it
+ * loads), and checking them one after another made every host wait for the others' answers.
+ */
+declare const SEO_RESOURCE_HOSTS_AT_ONCE = 4;
+/**
+ * READS A LIST OF CHECKS POLITELY: each host one check at a time, never two requests to one host
+ * closer than `delayMs` (a check that needs a second request waits again), up to
+ * {@link SEO_RESOURCE_HOSTS_AT_ONCE} hosts at once, so a host is never slowed by another's pause or
+ * answer. A shared CDN host (`SEO_SHARED_CDN_HOSTS`), built for every site's traffic, is asked at its
+ * own shorter pause; a site's own hosts keep `delayMs`. On the server every request takes its turn in
+ * the per-host pace every crawl of that host shares.
+ *
+ * @param targets - The URLs to check.
+ * @param userAgent - The crawler's User-Agent.
+ * @param deps - The guarded transport, the clock and the pause.
+ * @param options - The pause between two requests to one host and when to stop.
+ * @returns The readings in the list's order, of the checks made before the list stopped.
+ */
+declare function readResources(targets: readonly SeoResourceCheckTarget[], userAgent: string, deps: SeoResourceCheckDeps, options: SeoReadResourcesOptions): Promise<SeoResourceReading[]>;
+
+export { SEO_COMPRESS_MIN_BYTES, SEO_IMAGE_SIZE_MIN_BYTES, SEO_RESOURCE_HOSTS_AT_ONCE, readResources, seoCompressible };
+export type { SeoReadResourcesOptions, SeoResourceCheckDeps, SeoResourceCheckTarget, SeoResourceReading };

package/dist/resource-check.js

@@ -0,0 +1,3 @@
+export { SEO_COMPRESS_MIN_BYTES, SEO_IMAGE_SIZE_MIN_BYTES, SEO_RESOURCE_HOSTS_AT_ONCE, readResources, seoCompressible } from './chunk-J4ZNQFJO.js';
+import './chunk-7SGWGESF.js';
+import './chunk-6QPF6O7D.js';