@queryrev/ai-readiness 0.0.0-stage → 1.0.1

package/dist/cli/args.d.ts

@@ -0,0 +1,27 @@
+type OptionKind = 'boolean' | 'string';
+interface OptionSpec {
+    readonly type: OptionKind;
+    readonly short?: string;
+}
+export type CommandName = 'check' | 'robots test' | 'llms validate' | 'llms generate' | 'crawlers';
+interface CommandSpec {
+    readonly name: CommandName;
+    readonly options: Readonly<Record<string, OptionSpec>>;
+    readonly positionals: {
+        readonly min: number;
+        readonly max: number;
+        readonly label: string;
+    };
+}
+export declare const COMMANDS: Readonly<Record<CommandName, CommandSpec>>;
+export interface ParsedCommand {
+    readonly command: CommandName;
+    readonly positionals: readonly string[];
+    readonly values: Readonly<Record<string, string | boolean | undefined>>;
+}
+export declare function parseCommand(argv: readonly string[], offersPrivateNetwork?: boolean): ParsedCommand;
+export type FailOn = 'fail' | 'warn';
+export declare function failOnOf(parsed: ParsedCommand): FailOn;
+export declare const flag: (parsed: ParsedCommand, name: string) => boolean;
+export declare const text: (parsed: ParsedCommand, name: string) => string | undefined;
+export {};

package/dist/cli/bin.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli/commands/check.d.ts

@@ -0,0 +1,4 @@
+import { type ParsedCommand } from '../args.js';
+import { makeStyle } from '../render.js';
+import { type CliContext, type CommandResult } from '../types.js';
+export declare function runCheck(context: CliContext, parsed: ParsedCommand, style: ReturnType<typeof makeStyle>): Promise<CommandResult>;

package/dist/cli/commands/crawlers.d.ts

@@ -0,0 +1,3 @@
+import { type Style } from '../render.js';
+import { type CommandResult } from '../types.js';
+export declare function runCrawlers(style: Style): CommandResult;

package/dist/cli/commands/llms-generate.d.ts

@@ -0,0 +1,4 @@
+import { type ParsedCommand } from '../args.js';
+import { type Style } from '../render.js';
+import { type CliContext, type CommandResult } from '../types.js';
+export declare function runLlmsGenerate(context: CliContext, parsed: ParsedCommand, style: Style): Promise<CommandResult>;

package/dist/cli/commands/llms-validate.d.ts

@@ -0,0 +1,4 @@
+import { type ParsedCommand } from '../args.js';
+import { type Style } from '../render.js';
+import { type CliContext, type CommandResult } from '../types.js';
+export declare function runLlmsValidate(context: CliContext, parsed: ParsedCommand, style: Style): Promise<CommandResult>;

package/dist/cli/commands/robots-test.d.ts

@@ -0,0 +1,4 @@
+import { type ParsedCommand } from '../args.js';
+import { type Style } from '../render.js';
+import { type CliContext, type CommandResult } from '../types.js';
+export declare function runRobotsTest(context: CliContext, parsed: ParsedCommand, style: Style): Promise<CommandResult>;

package/dist/cli/common.d.ts

@@ -0,0 +1,27 @@
+import { type TextFetchResult } from '../net/fetch-text.js';
+import type { Finding } from '../facts/types.js';
+import type { FailOn } from './args.js';
+import { type CliContext, type ExitCode } from './types.js';
+/** 1 when a finding is at or above the threshold, else 0. */
+export declare function exitForFindings(findings: readonly Finding[], failOn: FailOn): ExitCode;
+/** Why a fetch gave no usable answer, in a few plain words. Empty for an answer. */
+export declare function describeFetchFailure(result: TextFetchResult, offersPrivateNetwork?: boolean): string;
+export interface Source {
+    readonly kind: 'file' | 'url';
+    /** The path, or the address. */
+    readonly location: string;
+}
+/**
+ * Tells a file from a site: an http(s) address is a site, a name that is an existing file is a
+ * file, and a bare host name (`example.com`) is a site.
+ */
+export declare function classifySource(context: CliContext, input: string, maxBytes: number): Promise<{
+    source: Source;
+    file: Awaited<ReturnType<CliContext['readFile']>> | null;
+}>;
+/** The address of a well-known file on a site: the given address, or `/name` at the site's root. */
+export declare function fileAddress(input: string, name: string): string;
+export declare const ROBOTS_READ_LIMIT: number;
+export declare const LLMS_READ_LIMIT: number;
+/** The text of a file read from disk; a byte order mark is left for the parsers to handle. */
+export declare function decodeFile(bytes: Uint8Array): string;

package/dist/cli/finding.d.ts

@@ -0,0 +1,3 @@
+import type { Finding, FindingLevel } from '../facts/types.js';
+/** A finding of the command line's own checks (a robots.txt test); the evidence is cut short. */
+export declare function cliFinding(id: string, level: FindingLevel, message: string, snippet?: string, anchor?: string): Finding;

package/dist/cli/main.d.ts

@@ -0,0 +1,7 @@
+import { type CliContext, type ExitCode } from './types.js';
+export declare const CLI_VERSION: string;
+/**
+ * Runs the command line. Returns the exit code and sets no global state: output goes through the
+ * context, so a test can run it with a fixture fetcher and read what it printed.
+ */
+export declare function runCli(argv: readonly string[], context: CliContext): Promise<ExitCode>;

package/dist/cli/node-context.d.ts

@@ -0,0 +1,3 @@
+import type { CliContext } from './types.js';
+/** The command line's real surroundings: this process's streams, environment and file system. */
+export declare function nodeContext(): CliContext;

package/dist/cli/render.d.ts

@@ -0,0 +1,26 @@
+import { crawlersCoveredLine } from '../crawlers/index.js';
+import type { Finding, FindingLevel } from '../facts/types.js';
+/** The "full report" link every human report ends with. It is printed, never opened. */
+export declare function fullReportUrl(command: string): string;
+/** What this tool cannot see, said once in the same words wherever it matters. */
+export declare const CANNOT_SEE = "Not seen: a CDN or firewall rule can answer AI crawlers differently from how it answers this tool.";
+export interface Style {
+    bold(text: string): string;
+    dim(text: string): string;
+    level(level: FindingLevel, text: string): string;
+}
+/** Colour only on a terminal, and never when NO_COLOR is set or `--no-color` is given. */
+export declare function makeStyle(options: {
+    readonly isTTY: boolean;
+    readonly env: Readonly<Record<string, string | undefined>>;
+    readonly noColorFlag: boolean;
+}): Style;
+export declare function summarize(findings: readonly Finding[]): Record<FindingLevel, number>;
+/** The findings as lines: a label, the id, the sentence, and the evidence under it. */
+export declare function renderFindings(findings: readonly Finding[], style: Style): string;
+export declare function renderSummary(findings: readonly Finding[]): string;
+/** The covered crawlers as a table: token, vendor, purpose. */
+export declare function renderCrawlerList(style: Style): string;
+export { crawlersCoveredLine };
+/** A table with a header row; cells are padded to the widest entry of their column. */
+export declare function renderTable(header: readonly string[], rows: readonly (readonly string[])[], style: Style): string;

package/dist/cli/run.d.ts

@@ -0,0 +1,11 @@
+/**
+ * The command runner (`@queryrev/ai-readiness/run`): the same code the command line runs, with
+ * the surroundings (streams, files, the fetcher) passed in. A program that offers the commands
+ * to another program, such as an MCP server, runs them here and prints exactly what
+ * `ai-readiness <command> --json` prints.
+ *
+ * It does not include the command line's real file system or its private-network fetcher: the
+ * caller builds the context.
+ */
+export { runCli, CLI_VERSION } from './main.js';
+export type { CliContext, ExitCode, FileRead, Writable } from './types.js';

package/dist/cli/sanitize.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Text that came from a site (a title, a heading, a line of its robots.txt) can carry terminal
+ * escape sequences and characters that reorder how text is shown. Every string of a result goes
+ * through here before it is printed or written as JSON.
+ */
+export declare function sanitizeDeep<T>(value: T): T;
+/** One line: whitespace runs, line breaks included, become one space. */
+export declare function oneLine(text: string): string;

package/dist/cli/types.d.ts

@@ -0,0 +1,59 @@
+import type { TextFetcher } from '../net/fetch-text.js';
+/**
+ * Exit codes, the contract of the command line:
+ *
+ * - 0: nothing at or above `--fail-on` (`fail` unless asked otherwise)
+ * - 1: the checks failed
+ * - 2: usage error (a wrong command, option or input file)
+ * - 3: the site could not be read (refused, unreachable, timed out, or robots.txt unreadable)
+ * - 4: internal error
+ */
+export type ExitCode = 0 | 1 | 2 | 3 | 4;
+export declare const EXIT_OK = 0;
+export declare const EXIT_CHECKS_FAILED = 1;
+export declare const EXIT_USAGE = 2;
+export declare const EXIT_UNREADABLE = 3;
+export declare const EXIT_INTERNAL = 4;
+export interface Writable {
+    write(chunk: string): unknown;
+    readonly isTTY?: boolean;
+}
+export interface FileRead {
+    readonly bytes: Uint8Array;
+    /** The file was longer than the limit and was cut there. */
+    readonly truncated: boolean;
+}
+/** Everything the command line touches outside its own code, so a test can replace it. */
+export interface CliContext {
+    readonly stdout: Writable;
+    readonly stderr: Writable;
+    readonly env: Readonly<Record<string, string | undefined>>;
+    /** Reads at most `maxBytes` of a file; throws an error with a `code` (`ENOENT`) when it cannot. */
+    readonly readFile: (path: string, maxBytes: number) => Promise<FileRead>;
+    /** Writes a file; without `overwrite` it fails when the file exists. */
+    readonly writeFile: (path: string, text: string, overwrite: boolean) => Promise<void>;
+    /**
+     * Whether the commands offer `--allow-private-network` (default true). A program that embeds
+     * the commands and has no such option sets this to false: the flag is then an unknown option
+     * and no message mentions it.
+     */
+    readonly offersPrivateNetwork?: boolean;
+    /** One fetcher per run; the command line closes it. */
+    readonly createFetcher: (options: {
+        allowPrivateNetwork: boolean;
+    }) => TextFetcher;
+}
+/** What a command returns; `runCli` prints it. */
+export interface CommandResult {
+    readonly exit: ExitCode;
+    /** The `--json` document. */
+    readonly json: unknown;
+    /** The human report, for standard output. */
+    readonly text: string;
+    /** Notes for standard error in the human mode (a generated file goes to standard output). */
+    readonly textErr?: string;
+}
+/** A wrong command, option or input: exit code 2. */
+export declare class UsageError extends Error {
+    readonly name = "UsageError";
+}

package/dist/cli/usage.d.ts

@@ -0,0 +1 @@
+export declare const USAGE = "ai-readiness: checks how a site's public files treat AI crawlers.\n\nUsage: ai-readiness <command> [arguments] [options]\n\nCommands\n  check <url>                       robots.txt access of the covered AI crawlers, llms.txt, the\n                                    home page as served, structured data and the sitemap\n  robots test <file|url>            which robots.txt line decides for a crawler and a path\n      --agent <token>                 one crawler token (any token); all covered crawlers without it\n      --path <path>                   the path to test, default /\n  llms validate <file|url>          checks an llms.txt against llmstxt.org\n      --links                         also request up to 20 links on the same site\n      --base <url>                    the site of a local file's links (needed with --links)\n  llms generate <url>               builds the skeleton of an llms.txt from the site's own pages\n      --max-pages <n>                 pages to list, 1 to 100, default 30\n      --out <file>                    write the file here (--force replaces an existing file)\n  crawlers                          the AI crawlers covered, with vendor and purpose\n\nOptions\n  --json                 one JSON document with schemaVersion 1, no colour, no other output\n  --fail-on <level>      exit 1 for findings at \"fail\" (default) or at \"warn\" (check, robots test, llms validate)\n  --allow-private-network\n                         reach a staging site on a private network (never cloud metadata addresses)\n  --no-color             no colour (also off without a terminal, or when NO_COLOR is set)\n  -h, --help             this text\n  -v, --version          the version\n\nExit codes\n  0  nothing at or above --fail-on\n  1  checks failed\n  2  usage error\n  3  the site could not be read (refused, unreachable, timed out)\n  4  internal error\n\nReads public files only. Sends nothing to queryrev.com.\n";

package/dist/cli.js

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+import { n as runCli } from "./chunks/main-Bd0uzoHe.js";
+import { n as createCliTextFetcher } from "./chunks/cli-BK65ic84.js";
+import { argv, env, stderr, stdout } from "node:process";
+import { open, writeFile } from "node:fs/promises";
+// src/cli/node-context.ts
+async function readFile(path, maxBytes) {
+	const handle = await open(path, "r");
+	try {
+		const buffer = Buffer.alloc(maxBytes + 1);
+		const { bytesRead } = await handle.read(buffer, 0, maxBytes + 1, 0);
+		return {
+			bytes: buffer.subarray(0, Math.min(bytesRead, maxBytes)),
+			truncated: bytesRead > maxBytes
+		};
+	} finally {
+		await handle.close();
+	}
+}
+/** The command line's real surroundings: this process's streams, environment and file system. */
+function nodeContext() {
+	return {
+		stdout,
+		stderr,
+		env,
+		readFile,
+		writeFile: (path, text, overwrite) => writeFile(path, text, {
+			encoding: "utf8",
+			flag: overwrite ? "w" : "wx"
+		}),
+		createFetcher: ({ allowPrivateNetwork }) => createCliTextFetcher({ allowPrivateNetwork })
+	};
+}
+// src/cli/bin.ts
+process.exitCode = await runCli(argv.slice(2), nodeContext());
+export {};

package/dist/crawlers/index.d.ts

@@ -0,0 +1,79 @@
+import { type RobotsTxt, type RuleMatch } from '../robots/index.js';
+/**
+ * Which AI crawlers are measured, and what a robots.txt says to each of them. Nothing outside
+ * `STUDY_AGENTS` is measured or named.
+ */
+/**
+ * The product tokens measured, and the catch-all. The study names the vendors it measures on its
+ * page; any crawler not listed here is neither measured nor named.
+ */
+export declare const STUDY_AGENTS: readonly ["GPTBot", "OAI-SearchBot", "ChatGPT-User", "PerplexityBot", "Perplexity-User", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Amazonbot", "Bytespider", "*"];
+export type StudyAgent = (typeof STUDY_AGENTS)[number];
+export type AiAgent = Exclude<StudyAgent, '*'>;
+/** The AI agents (every measured token but the catch-all). */
+export declare const AI_AGENTS: AiAgent[];
+/**
+ * What the rules of a robots.txt mean for one agent, read with the group that applies to it
+ * (its own named group, otherwise the `*` group; RFC 9309 2.2.1):
+ *
+ * - `blocked`: the whole site is closed to it: a rule for the whole site (`/`, `/*`, `*`)
+ *   disallows `/` and no narrower `Allow` rule opens any other path.
+ * - `partial`: some of the site is closed to it: `/` is disallowed but an `Allow` rule opens some
+ *   paths, or `/` is disallowed only by a narrower rule (`Disallow: /$` closes the home page,
+ *   not the site); or `/` is open but the group that applies also disallows the whole site (an
+ *   `Allow: /$` exception to `Disallow: /`); or the agent has a group of its own that disallows
+ *   a path the `*` group does not. (Paths a site keeps from every crawler, under `*`, are
+ *   housekeeping: an agent that inherits them, or whose own group repeats them, is `allowed`.)
+ * - `allowed`: anything else, a missing robots.txt included.
+ *
+ * `named` tells a rule written for the agent from one it inherits from `*`.
+ */
+export type AgentAccess = 'allowed' | 'partial' | 'blocked';
+export interface AgentVerdict {
+    readonly access: AgentAccess;
+    /** The robots.txt has a group naming this agent's product token (never true for `*`). */
+    readonly named: boolean;
+    /** Whether the rules that apply to the agent let it fetch `/llms.txt`. */
+    readonly llmsTxt: boolean;
+}
+export declare function agentVerdict(robots: RobotsTxt, agent: string): AgentVerdict;
+/** The verdicts of every measured agent for one robots.txt. */
+export declare function verdictsFor(robots: RobotsTxt): Record<StudyAgent, AgentVerdict>;
+/** What a crawler is for: collecting pages for AI training, indexing for AI search, or fetching a page when a user asks. */
+export type CrawlerPurpose = 'training' | 'search' | 'user';
+/** The purposes, in the order they are listed. */
+export declare const CRAWLER_PURPOSES: readonly CrawlerPurpose[];
+export interface CrawlerInfo {
+    /** The product token of the crawler, as it is written in a `User-agent` line. */
+    readonly token: AiAgent;
+    readonly vendor: string;
+    readonly purpose: CrawlerPurpose;
+}
+/** The crawlers covered, in the order of `AI_AGENTS`. No other crawler is measured or named. */
+export declare const CRAWLERS: readonly CrawlerInfo[];
+/**
+ * The scope line every report carries: which crawlers were checked. A report never claims
+ * to cover every AI crawler.
+ */
+export declare function crawlersCoveredLine(): string;
+/** What a robots.txt says to one covered crawler, for the site and for one path. */
+export interface CrawlerAccess {
+    readonly crawler: CrawlerInfo;
+    /** The verdict for the whole site (see `agentVerdict`). */
+    readonly access: AgentAccess;
+    readonly named: boolean;
+    /** The rule that decides `/`. */
+    readonly root: RuleMatch;
+    /** The rule that decides the path asked about; null when no path other than `/` was asked. */
+    readonly path: RuleMatch | null;
+    /** Whether the crawler may fetch `/llms.txt`. */
+    readonly llmsTxt: boolean;
+}
+/**
+ * The access of every covered crawler: the verdict for the site, and the deciding rule for `/`
+ * and for `path` (a path and query, such as `/blog/post?x=1`). A missing robots.txt is an
+ * empty one: pass `parseRobots('')`.
+ */
+export declare function crawlerAccess(robots: RobotsTxt, options?: {
+    path?: string;
+}): CrawlerAccess[];

package/dist/facts/app-shell.d.ts

@@ -0,0 +1,24 @@
+/**
+ * The empty application shell: an HTML document over a few KiB that carries almost no text and
+ * an application root element, or asks the reader to enable JavaScript. The constants are the
+ * ones the site crawler's render decision uses (it imports them from here), so a page is called
+ * a shell by one rule everywhere.
+ */
+/** An empty shell is HTML over this many bytes (5 KB, read as 5 KiB)... */
+export declare const EMPTY_SHELL_MIN_BYTES: number;
+/** ...with fewer than this many words of main text... */
+export declare const EMPTY_SHELL_MAX_WORDS = 50;
+/**
+ * ...and an application root element. Ids that the common frameworks mount into: React
+ * (create-react-app, Vite), Vue, Next.js, Nuxt, Gatsby, SvelteKit, Quasar, Remix, Docusaurus.
+ */
+export declare const APP_ROOT_IDS: readonly string[];
+/** Elements and attributes that mark an application root besides the ids (Angular, AngularJS, React, Vue SSR). */
+export declare const APP_ROOT_TAG = "app-root";
+export declare const APP_ROOT_ATTRIBUTES: readonly string[];
+/** The same marks as one CSS selector, for a reader that has a DOM. */
+export declare const APP_ROOT_SELECTOR: string;
+/** "Enable JavaScript" notices, in the words sites use. */
+export declare const JS_NOTICE: RegExp;
+/** Whether one start tag (its name and its attributes by name) is an application root. */
+export declare function isAppRoot(name: string, attributes: ReadonlyMap<string, string>): boolean;

package/dist/facts/check-site.d.ts

@@ -0,0 +1,13 @@
+import { type TextFetchResult } from '../net/fetch-text.js';
+import { type RobotsTxt } from '../robots/index.js';
+import type { CheckOptions, RobotsRead, SiteCheck } from './types.js';
+export declare function robotsRead(result: TextFetchResult, url: string): {
+    read: RobotsRead;
+    robots: RobotsTxt | null;
+};
+/**
+ * Checks a site given as a host name (`example.com`) or an http(s) address (only its scheme and
+ * host are used). Never throws for what a site does or for an address that cannot be checked:
+ * the result says what was read and what was not.
+ */
+export declare function checkSite(input: string, options?: CheckOptions): Promise<SiteCheck>;

package/dist/facts/domain.d.ts

@@ -0,0 +1,15 @@
+/**
+ * A registrable domain without a public-suffix list. The library keeps one runtime dependency,
+ * so the built-in rule is small: the last two labels, or the last three when the last two are a
+ * common second-level suffix (`co.uk`, `com.au`). A caller that needs the full list passes its
+ * own function (`CheckOptions.registrableDomain`, `ReadHomePageOptions.registrableDomain`).
+ */
+/** A host name as comparisons use it: lower case, without one trailing dot. */
+export declare function normalizeHost(host: string): string;
+/**
+ * The registrable domain of a host name (`www.shop.example.co.uk` → `example.co.uk`). A name
+ * with one label, or an IP literal, is returned as it is.
+ */
+export declare function registrableDomain(host: string): string;
+/** Whether `host` is the registrable domain `domain` or a name under it. */
+export declare function isOnDomain(host: string, domain: string): boolean;

package/dist/facts/findings.d.ts

@@ -0,0 +1,5 @@
+import type { Finding, SiteCheck } from './types.js';
+/** The finding ids that exist, for tests and for tools that list them. */
+export declare const FINDING_IDS: readonly string[];
+/** The findings for the facts of a check. */
+export declare function buildFindings(site: Omit<SiteCheck, 'findings'>): Finding[];

package/dist/facts/home-page.d.ts

@@ -0,0 +1,18 @@
+import type { HomePageFacts } from './types.js';
+/** A home page with at least this many words of main text has "text in the HTML". */
+export declare const TEXT_IN_HTML_MIN_WORDS = 150;
+export interface ReadHomePageOptions {
+    /** The values of the `X-Robots-Tag` response header, if any. */
+    readonly xRobotsTag?: string | readonly string[] | null;
+    /** Maps a host name to its registrable domain; the built-in rule knows common suffixes only. */
+    readonly registrableDomain?: (host: string) => string;
+}
+/**
+ * Whether `X-Robots-Tag` values say noindex to every crawler. A value may start with an agent
+ * (`googlebot: noindex, nofollow`), which then applies to the rest of that value up to the next
+ * agent; such a rule is for one crawler and does not count. `unavailable_after: <date>` is a
+ * directive, not an agent.
+ */
+export declare function xRobotsNoindex(values: string | readonly string[] | null | undefined): boolean;
+/** Reads the facts of a home page. Never throws. */
+export declare function readHomePage(html: string, url: string, options?: ReadHomePageOptions): HomePageFacts;

package/dist/facts/home-response.d.ts

@@ -0,0 +1,5 @@
+import type { TextFetchResult } from '../net/fetch-text.js';
+import { type ReadHomePageOptions } from './home-page.js';
+import type { HomeRead } from './types.js';
+/** Reads the answer for the home page. `not_allowed` is not decided here: no request was made. */
+export declare function readHomeResponse(result: TextFetchResult, options?: Pick<ReadHomePageOptions, 'registrableDomain'>): HomeRead;

package/dist/facts/html.d.ts

@@ -0,0 +1,37 @@
+import type { JsonLdBound } from './types.js';
+/** Characters of a page scanned; the fetch cap for a page is 2 MiB, so a fetched page fits. */
+export declare const HOME_SCAN_MAX_CHARS: number;
+/** JSON-LD limits: blocks read, one block's size, all blocks' size (in characters). */
+export declare const MAX_LD_BLOCKS = 50;
+export declare const MAX_LD_BLOCK_CHARS: number;
+export declare const MAX_LD_TOTAL_CHARS: number;
+/** What one pass over a page found. */
+export interface PageScan {
+    /** Text of `<main>` and `[role=main]`; empty when `hasMain` is false. */
+    readonly mainText: string;
+    /** Text of the body without `header`, `nav`, `footer` and `aside`. */
+    readonly bodyText: string;
+    readonly hasMain: boolean;
+    /** What a visitor sees (no scripts, styles, templates), cut to 64 KiB. */
+    readonly visibleText: string;
+    /** Text inside `<noscript>` and `<title>`, cut to 64 KiB. */
+    readonly noticeText: string;
+    /** `content` of every `<meta name="robots">`. */
+    readonly metaRobots: readonly string[];
+    /** `itemtype` values, split on spaces, at most 200. */
+    readonly itemtypes: readonly string[];
+    /** Contents of the `<script type="application/ld+json">` blocks that were read. */
+    readonly ldBlocks: readonly string[];
+    /** ld+json scripts found, whether or not they were read. */
+    readonly ldScripts: number;
+    readonly ldBounds: readonly JsonLdBound[];
+    /** The input was longer than `HOME_SCAN_MAX_CHARS`. */
+    readonly truncated: boolean;
+}
+/** Scans a page. Never throws; text it cannot make sense of is left out. */
+export declare function scanPage(input: string): PageScan;
+/**
+ * Whether the page has an application root element (see `isAppRoot`): a second, light pass over
+ * the start tags that stops at the first one found.
+ */
+export declare function hasAppRoot(input: string): boolean;

package/dist/facts/index.d.ts

@@ -0,0 +1,16 @@
+export { APP_ROOT_ATTRIBUTES, APP_ROOT_IDS, APP_ROOT_SELECTOR, APP_ROOT_TAG, EMPTY_SHELL_MAX_WORDS, EMPTY_SHELL_MIN_BYTES, isAppRoot, JS_NOTICE, } from './app-shell.js';
+export { checkSite } from './check-site.js';
+export { registrableDomain } from './domain.js';
+export { buildFindings, FINDING_IDS } from './findings.js';
+export { readHomeResponse } from './home-response.js';
+export { readHomePage, TEXT_IN_HTML_MIN_WORDS, xRobotsNoindex, type ReadHomePageOptions, } from './home-page.js';
+export { CHALLENGE_MARKS, INTERSTITIAL_MARKS, CHALLENGE_MAX_WORDS, HTML_MARKUP, isChallenge, isHomeChallenge, PLACEHOLDER_MARKS, PLACEHOLDER_MAX_WORDS, placeholderMark, } from './marks.js';
+export { PROFILE_HOSTS, PROFILE_KINDS, profileKind } from './profiles.js';
+export { READER_TOKEN } from './reader.js';
+export { SCHEMA_ORG_TYPES } from './schema-org-types.js';
+export { ORGANIZATION_TYPES, SCHEMA_ORG_SHA256, SCHEMA_ORG_VERSION, } from './schema-org-organization-types.js';
+export { MAX_LOCS_COUNTED, readSitemap, readSitemapText, SITEMAP_MAX_BYTES, type ReadSitemapOptions, } from './sitemap.js';
+export type { CheckOptions, CrawlerRootAccess, Finding, FindingLevel, HomeContent, HomeOutcome, HomePageFacts, HomeRead, JsonLdBound, JsonLdFacts, LlmsRead, OrganizationFacts, OrganizationSource, ProfileKind, RobotsOutcome, RobotsRead, SiteCheck, SitemapFacts, SitemapOutcome, SitemapRead, SitemapReason, SitemapSource, } from './types.js';
+export { countWords, MAX_WORDS_COUNTED, type WordCount } from './words.js';
+export { HOME_SCAN_MAX_CHARS, MAX_LD_BLOCK_CHARS, MAX_LD_BLOCKS, MAX_LD_TOTAL_CHARS } from './html.js';
+export { MAX_LD_DEPTH, MAX_LD_NODES } from './json-ld.js';

package/dist/facts/json-ld.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Reads the JSON-LD blocks of a page: parses them, keeps those whose top-level `@context` names
+ * schema.org, and walks them for typed nodes. The walk is bounded: depth 16, 5,000 nodes. Past
+ * a bound the rest is not read and the bound is reported.
+ */
+import type { JsonLdBound } from './types.js';
+export declare const MAX_LD_DEPTH = 16;
+export declare const MAX_LD_NODES = 5000;
+type Json = unknown;
+export type JsonObject = Record<string, Json>;
+/** A node of the data that has an `@type`. */
+export interface TypedNode {
+    /** Type names with the schema.org prefixes removed. */
+    readonly types: readonly string[];
+    readonly node: JsonObject;
+}
+export interface JsonLdWalk {
+    readonly scripts: number;
+    readonly parsed: number;
+    readonly schemaOrgBlocks: number;
+    readonly errors: number;
+    readonly bounds: readonly JsonLdBound[];
+    /** Typed nodes of the blocks that name schema.org, in document order (parents first). */
+    readonly typed: readonly TypedNode[];
+}
+/** Whether an `@context` value names schema.org (as a string, in an array, or as `@vocab`). */
+export declare function namesSchemaOrg(context: Json): boolean;
+export declare function isObject(value: Json): value is JsonObject;
+/** A type name without `schema:` and `http(s)://schema.org/`. */
+export declare function schemaTypeName(raw: string): string;
+/** The block's text without a wrapping `<!-- -->` or `<![CDATA[ ]]>`, and without a byte order mark. */
+export declare function unwrapBlock(text: string): string;
+/** Parses the blocks and walks the data that names schema.org. */
+export declare function walkJsonLd(blocks: readonly string[], scripts: number, inheritedBounds: readonly JsonLdBound[]): JsonLdWalk;
+export {};

package/dist/facts/marks.d.ts

@@ -0,0 +1,51 @@
+/** HTML markup anywhere in a body: no robots.txt holds it. */
+export declare const HTML_MARKUP: RegExp;
+/**
+ * The marks of a bot check served in place of a file: Cloudflare ("Just a moment…", its
+ * challenge platform), reCAPTCHA, hCaptcha, Akamai Bot Manager, HUMAN (PerimeterX), DataDome,
+ * Imperva (Incapsula), AWS WAF, Sucuri, a hosting platform's security checkpoint, Kasada, and the
+ * generic wording.
+ */
+export declare const CHALLENGE_MARKS: RegExp;
+/**
+ * Whether a 2xx answer at /robots.txt is not the file: a 202 (a check still running, most
+ * often), or a body with no robots.txt group that is an HTML page or carries the marks of a bot
+ * check. Such a site is "not read", never "read, no rules". A body with at least one group is a
+ * file whatever its content type says.
+ */
+export declare function isChallenge(status: number | null, body: string): boolean;
+/** A page with fewer words than this, carrying an interstitial mark, is a bot check. */
+export declare const CHALLENGE_MAX_WORDS = 100;
+/** A page with fewer words than this, carrying a mark, is a placeholder. */
+export declare const PLACEHOLDER_MAX_WORDS = 150;
+/** Characters of a page searched for marks: the head and the start of the body. */
+export declare const MARKS_SCAN_CHARS: number;
+/**
+ * What a bot-check page says to a visitor, in its title or visible text: Cloudflare's "Just a
+ * moment..." and "One more step", "Checking your browser", Imperva's "Pardon Our Interruption",
+ * "Verify you are human", "Access denied" beside a captcha, a security checkpoint, and the
+ * generic wording. These are for the home page, matched against text a visitor sees, never
+ * against script sources: the scripts of a bot-check vendor (`challenge-platform`, a captcha
+ * library, `datadome`) are loaded on ordinary pages too, and an empty application shell that
+ * loads one is a page, not a check.
+ */
+export declare const INTERSTITIAL_MARKS: RegExp;
+/**
+ * Whether the home page is a bot check: a 202, or a small page (under `CHALLENGE_MAX_WORDS`
+ * words of text) whose title or visible text (`text`) carries one of `INTERSTITIAL_MARKS`.
+ */
+export declare function isHomeChallenge(status: number | null, text: string, mainWords: number): boolean;
+/**
+ * The marks of a placeholder home page, each a fixed phrase: the default pages of common web
+ * servers and control panels, and the wording of domain parking pages. Matched without case
+ * against the start of the page's HTML.
+ */
+export declare const PLACEHOLDER_MARKS: readonly {
+    readonly id: string;
+    readonly pattern: RegExp;
+}[];
+/**
+ * Whether the home page is a placeholder: a small page (under `PLACEHOLDER_MAX_WORDS` words of
+ * text) that carries one of `PLACEHOLDER_MARKS`. Returns the mark's id, or null.
+ */
+export declare function placeholderMark(html: string, mainWords: number): string | null;

package/dist/facts/organization.d.ts

@@ -0,0 +1,23 @@
+/**
+ * The Organization of a home page: which typed nodes are one, which is the primary node, and
+ * what it says about itself.
+ */
+import { type JsonObject, type TypedNode } from './json-ld.js';
+import type { OrganizationFacts } from './types.js';
+/** Whether a (prefix-free) type name is Organization or one of its schema.org subtypes. */
+export declare function isOrganizationType(name: string): boolean;
+export interface OrganizationNode {
+    readonly type: string;
+    readonly node: JsonObject;
+}
+/** The Organization nodes among the typed nodes, in document order. */
+export declare function organizationNodes(typed: readonly TypedNode[]): OrganizationNode[];
+/**
+ * The primary node: the first Organization node whose `url` is on the site's registrable
+ * domain, else the first. Null without any.
+ */
+export declare function primaryOrganization(nodes: readonly OrganizationNode[], siteDomain: string, registrable: (host: string) => string): OrganizationNode | null;
+/** What the primary node says about itself. */
+export declare function describeOrganization(primary: OrganizationNode, pageUrl: string, siteDomain: string, registrable: (host: string) => string): OrganizationFacts;
+/** Whether any `potentialAction` of the node is a `SearchAction`. */
+export declare function hasSearchAction(node: JsonObject): boolean;

package/dist/facts/origin-robots.d.ts

@@ -0,0 +1,20 @@
+/**
+ * robots.txt per origin. An address on a subdomain, or on the other scheme, is a different
+ * origin with its own robots.txt: before this tool requests anything there, it reads that
+ * origin's file once and asks whether its own token may fetch the path. Answers are kept for
+ * the run.
+ */
+import { type RobotsTxt } from '../robots/index.js';
+import { type TextFetcher } from '../net/fetch-text.js';
+export type OriginAccess = 'allowed' | 'closed' | 'unreadable';
+export interface OriginRobots {
+    /** The parsed robots.txt of an origin (a missing file is an empty one); null when unreadable. */
+    robotsOf(origin: string): Promise<RobotsTxt | null>;
+    /** Whether this tool's token may fetch the address, by the robots.txt of the address's origin. */
+    check(url: string): Promise<OriginAccess>;
+}
+export declare function createOriginRobots(options: {
+    readonly fetcher: TextFetcher;
+    readonly signal?: AbortSignal;
+    readonly redirectAllowed?: (target: URL) => boolean;
+}): OriginRobots;

package/dist/facts/profiles.d.ts

@@ -0,0 +1,13 @@
+import type { ProfileKind } from './types.js';
+/**
+ * Profile kinds of a `sameAs` link, by host. The list is part of the method: a host is the
+ * domain or any name under it (`en.wikipedia.org`, `m.facebook.com`).
+ */
+export declare const PROFILE_HOSTS: readonly {
+    readonly kind: Exclude<ProfileKind, 'other'>;
+    readonly hosts: readonly string[];
+}[];
+/** The kinds in the order results list them. */
+export declare const PROFILE_KINDS: readonly ProfileKind[];
+/** The kind of the profile at a host name. */
+export declare function profileKind(host: string): ProfileKind;