@queryrev/ai-readiness 0.0.0-stage → 1.0.0

package/dist/chunks/main-CJu6SE-W.js

@@ -0,0 +1,1491 @@
+import { $ as looksLikeHtml, F as sitemapLocations, I as stripUnsafeCharacters, M as attributes, N as pageMeta, P as renderLlmsTxt, X as registrableDomain, _t as parseRobots, a as robotsRead, ct as crawlerAccess, f as READER_TOKEN, g as xRobotsNoindex, gt as matchRobots, ht as isAllowedByRobots, i as checkSite, it as CRAWLERS, j as PAGE_SCAN_MAX_CHARS, lt as crawlersCoveredLine, mt as crawlDelayFor, n as llmsFinding, nt as validateLlmsTxt, r as llmsFindings } from "./findings-_twhMw3_.js";
+import { a as GENERATE_BUDGET_MS, i as FETCH_LIMITS, m as createTextFetcher, n as CHECK_BUDGET_MS, p as checkTextUrl } from "./fetch-text-NUUnM91O.js";
+import { parseArgs } from "node:util";
+import { gunzipSync } from "node:zlib";
+// src/cli/types.ts
+/** A wrong command, option or input: exit code 2. */
+var UsageError = class extends Error {
+	name = "UsageError";
+};
+// src/cli/args.ts
+var COMMON = {
+	json: { type: "boolean" },
+	"no-color": { type: "boolean" },
+	help: {
+		type: "boolean",
+		short: "h"
+	}
+};
+var FAIL_ON = { "fail-on": { type: "string" } };
+var PRIVATE = { "allow-private-network": { type: "boolean" } };
+var COMMANDS = {
+	check: {
+		name: "check",
+		options: {
+			...COMMON,
+			...FAIL_ON,
+			...PRIVATE
+		},
+		positionals: {
+			min: 1,
+			max: 1,
+			label: "<url>"
+		}
+	},
+	"robots test": {
+		name: "robots test",
+		options: {
+			...COMMON,
+			...FAIL_ON,
+			...PRIVATE,
+			agent: { type: "string" },
+			path: { type: "string" }
+		},
+		positionals: {
+			min: 1,
+			max: 1,
+			label: "<file|url>"
+		}
+	},
+	"llms validate": {
+		name: "llms validate",
+		options: {
+			...COMMON,
+			...FAIL_ON,
+			...PRIVATE,
+			links: { type: "boolean" },
+			base: { type: "string" }
+		},
+		positionals: {
+			min: 1,
+			max: 1,
+			label: "<file|url>"
+		}
+	},
+	"llms generate": {
+		name: "llms generate",
+		options: {
+			...COMMON,
+			...PRIVATE,
+			"max-pages": { type: "string" },
+			out: { type: "string" },
+			force: { type: "boolean" }
+		},
+		positionals: {
+			min: 1,
+			max: 1,
+			label: "<url>"
+		}
+	},
+	crawlers: {
+		name: "crawlers",
+		options: { ...COMMON },
+		positionals: {
+			min: 0,
+			max: 0,
+			label: ""
+		}
+	}
+};
+/** The command words at the start of the arguments, and how many of them there are. */
+function commandOf(argv) {
+	const first = argv[0];
+	if (first === void 0 || first.startsWith("-")) throw new UsageError("Give a command: check, robots test, llms validate, llms generate or crawlers.");
+	if (first === "robots" || first === "llms") {
+		const second = argv[1];
+		const name = `${first} ${second ?? ""}`;
+		if (second === void 0 || !(name in COMMANDS)) throw new UsageError(first === "robots" ? "\"robots\" needs a subcommand: robots test <file|url>." : "\"llms\" needs a subcommand: llms validate <file|url> or llms generate <url>.");
+		return {
+			name,
+			words: 2
+		};
+	}
+	if (first in COMMANDS) return {
+		name: first,
+		words: 1
+	};
+	throw new UsageError(`Unknown command "${first.slice(0, 40)}".`);
+}
+function parseCommand(argv, offersPrivateNetwork = true) {
+	const { name, words } = commandOf(argv);
+	const spec = COMMANDS[name];
+	const specOptions = offersPrivateNetwork ? spec.options : Object.fromEntries(Object.entries(spec.options).filter(([key]) => !(key in PRIVATE)));
+	let parsed;
+	try {
+		parsed = parseArgs({
+			args: [...argv.slice(words)],
+			options: Object.fromEntries(Object.entries(specOptions).map(([key, option]) => [key, option.short === void 0 ? { type: option.type } : {
+				type: option.type,
+				short: option.short
+			}])),
+			allowPositionals: true,
+			strict: true
+		});
+	} catch (error) {
+		throw new UsageError(error instanceof Error ? error.message.split("\n")[0].slice(0, 200) : "Invalid arguments.");
+	}
+	const count = parsed.positionals.length;
+	if (parsed.values["help"] === true) return {
+		command: name,
+		positionals: parsed.positionals,
+		values: parsed.values
+	};
+	if (count < spec.positionals.min) throw new UsageError(`"${name}" needs ${spec.positionals.label}.`);
+	if (count > spec.positionals.max) throw new UsageError(`"${name}" takes ${spec.positionals.max === 0 ? "no argument" : `only ${spec.positionals.label}`}.`);
+	return {
+		command: name,
+		positionals: parsed.positionals,
+		values: parsed.values
+	};
+}
+function failOnOf(parsed) {
+	const value = parsed.values["fail-on"];
+	if (value === void 0) return "fail";
+	if (value === "fail" || value === "warn") return value;
+	throw new UsageError("--fail-on takes \"fail\" or \"warn\".");
+}
+var flag = (parsed, name) => parsed.values[name] === true;
+var text = (parsed, name) => {
+	const value = parsed.values[name];
+	return typeof value === "string" ? value : void 0;
+};
+// src/cli/common.ts
+/** 1 when a finding is at or above the threshold, else 0. */
+function exitForFindings(findings, failOn) {
+	return findings.some((finding) => finding.level === "fail" || failOn === "warn" && finding.level === "warn") ? 1 : 0;
+}
+var OUTCOME_TEXT = {
+	refused_url: "the address is not one this tool will request",
+	refused_address: "the address resolves to a private or reserved network",
+	dns_failed: "the host name does not resolve",
+	too_many_redirects: "too many redirects",
+	redirect_loop: "a redirect loop",
+	invalid_redirect: "a redirect without a usable address",
+	redirect_refused: "a redirect to another site, which was not followed",
+	timeout: "the request timed out",
+	budget_exceeded: "the time limit of the run was reached",
+	host_stopped: "the host asked this tool to stop (429 or 503)",
+	network_error: "the connection failed"
+};
+/** Why a fetch gave no usable answer, in a few plain words. Empty for an answer. */
+function describeFetchFailure(result, offersPrivateNetwork = true) {
+	if (result.outcome === "response") return result.status === null ? "no answer" : `status ${result.status}`;
+	const text = OUTCOME_TEXT[result.outcome] ?? "no answer";
+	return result.outcome === "refused_address" && offersPrivateNetwork ? `${text} (use --allow-private-network for a staging site)` : text;
+}
+/** A name that ends like a file is a file, even where the ending is also a country's domain. */
+var FILE_NAME = /\.(?:txt|md|markdown)$/i;
+var HOST_LIKE = /^(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}(?:[/?#].*)?$/i;
+/**
+* Tells a file from a site: an http(s) address is a site, a name that is an existing file is a
+* file, and a bare host name (`example.com`) is a site.
+*/
+async function classifySource(context, input, maxBytes) {
+	if (/^https?:\/\//i.test(input)) return {
+		source: {
+			kind: "url",
+			location: input
+		},
+		file: null
+	};
+	try {
+		const file = await context.readFile(input, maxBytes);
+		return {
+			source: {
+				kind: "file",
+				location: input
+			},
+			file
+		};
+	} catch (error) {
+		const code = error.code;
+		if (code === "ENOENT" && HOST_LIKE.test(input) && !FILE_NAME.test(input)) return {
+			source: {
+				kind: "url",
+				location: `https://${input}`
+			},
+			file: null
+		};
+		if (code === "ENOENT") throw new UsageError(`No such file: ${input.slice(0, 200)}`);
+		if (code === "EISDIR") throw new UsageError(`${input.slice(0, 200)} is a directory.`);
+		if (code === "EACCES" || code === "EPERM") throw new UsageError(`Cannot read ${input.slice(0, 200)}: permission denied.`);
+		throw new UsageError(`Cannot read ${input.slice(0, 200)}.`);
+	}
+}
+/** The address of a well-known file on a site: the given address, or `/name` at the site's root. */
+function fileAddress(input, name) {
+	let url;
+	try {
+		url = new URL(input);
+	} catch {
+		throw new UsageError(`Not a web address: ${input.slice(0, 200)}`);
+	}
+	if (url.pathname === "" || url.pathname === "/") {
+		url.pathname = `/${name}`;
+		url.search = "";
+	}
+	url.hash = "";
+	return url.href;
+}
+var ROBOTS_READ_LIMIT = FETCH_LIMITS.robots;
+var LLMS_READ_LIMIT = FETCH_LIMITS.llms;
+/** The text of a file read from disk; a byte order mark is left for the parsers to handle. */
+function decodeFile(bytes) {
+	return new TextDecoder("utf-8", { fatal: false }).decode(bytes);
+}
+// src/cli/sanitize.ts
+/**
+* Text that came from a site (a title, a heading, a line of its robots.txt) can carry terminal
+* escape sequences and characters that reorder how text is shown. Every string of a result goes
+* through here before it is printed or written as JSON.
+*/
+function sanitizeDeep(value) {
+	if (typeof value === "string") return stripUnsafeCharacters(value);
+	if (Array.isArray(value)) return value.map((item) => sanitizeDeep(item));
+	if (value !== null && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, sanitizeDeep(item)]));
+	return value;
+}
+/** One line: whitespace runs, line breaks included, become one space. */
+function oneLine(text) {
+	return stripUnsafeCharacters(text).replace(/\s+/g, " ").trim();
+}
+// src/cli/render.ts
+/** The "full report" link every human report ends with. It is printed, never opened. */
+function fullReportUrl(command) {
+	return `https://queryrev.com/tools?utm_source=ai-readiness-cli&utm_medium=cli&utm_campaign=full-report&utm_content=${command}`;
+}
+/** What this tool cannot see, said once in the same words wherever it matters. */
+var CANNOT_SEE = "Not seen: a CDN or firewall rule can answer AI crawlers differently from how it answers this tool.";
+var CODES = {
+	pass: "32",
+	warn: "33",
+	fail: "31",
+	info: "36"
+};
+/** Colour only on a terminal, and never when NO_COLOR is set or `--no-color` is given. */
+function makeStyle(options) {
+	const noColor = options.env["NO_COLOR"] !== void 0 && options.env["NO_COLOR"] !== "";
+	const on = options.isTTY && !noColor && !options.noColorFlag && options.env["TERM"] !== "dumb";
+	const wrap = (code, text) => on ? `\u001b[${code}m${text}\u001b[0m` : text;
+	return {
+		bold: (text) => wrap("1", text),
+		dim: (text) => wrap("2", text),
+		level: (level, text) => wrap(CODES[level], text)
+	};
+}
+var LABEL = {
+	pass: "PASS",
+	warn: "WARN",
+	fail: "FAIL",
+	info: "INFO"
+};
+function summarize(findings) {
+	const counts = {
+		pass: 0,
+		warn: 0,
+		fail: 0,
+		info: 0
+	};
+	for (const finding of findings) counts[finding.level] += 1;
+	return counts;
+}
+/** The findings as lines: a label, the id, the sentence, and the evidence under it. */
+function renderFindings(findings, style) {
+	const idWidth = Math.min(30, Math.max(0, ...findings.map((finding) => finding.id.length)));
+	const lines = [];
+	for (const finding of findings) {
+		lines.push(`  ${style.level(finding.level, LABEL[finding.level])}  ${finding.id.padEnd(idWidth)}  ${oneLine(finding.message)}`);
+		if (finding.snippet !== void 0) for (const line of finding.snippet.split(/\r\n|\r|\n/)) {
+			if (line.trim() === "") continue;
+			lines.push(`        ${style.dim("|")} ${style.dim(line.trim())}`);
+		}
+	}
+	return lines.join("\n");
+}
+function renderSummary(findings) {
+	const counts = summarize(findings);
+	return `${counts.pass} pass, ${counts.warn} warn, ${counts.fail} fail, ${counts.info} info`;
+}
+/** The covered crawlers as a table: token, vendor, purpose. */
+function renderCrawlerList(style) {
+	const width = Math.max(...CRAWLERS.map((crawler) => crawler.token.length));
+	const vendorWidth = Math.max(...CRAWLERS.map((crawler) => crawler.vendor.length));
+	return [`  ${style.bold("Crawler".padEnd(width))}  ${style.bold("Vendor".padEnd(vendorWidth))}  ${style.bold("Purpose")}`, ...CRAWLERS.map((crawler) => `  ${crawler.token.padEnd(width)}  ${crawler.vendor.padEnd(vendorWidth)}  ${crawler.purpose}`)].join("\n");
+}
+/** A table with a header row; cells are padded to the widest entry of their column. */
+function renderTable(header, rows, style) {
+	const widths = header.map((cell, column) => Math.max(cell.length, ...rows.map((row) => (row[column] ?? "").length)));
+	const line = (cells, bold) => `  ${cells.map((cell, column) => {
+		const padded = column === cells.length - 1 ? cell : cell.padEnd(widths[column] ?? 0);
+		return bold ? style.bold(padded) : padded;
+	}).join("  ")}`;
+	return [line(header, true), ...rows.map((row) => line(row, false))].join("\n");
+}
+// src/cli/commands/check.ts
+var PRIVATE_NETWORK_HINT = "A staging site on a private network is reached with --allow-private-network.";
+/** A site that gave no robots.txt answer to read, or an address that cannot be checked. */
+function unreadable(site) {
+	if (site.findings.some((finding) => finding.id === "site-refused")) return site.findings[0]?.snippet ?? "the address cannot be checked";
+	if (site.robots.outcome === "not_read") return site.robots.status === null ? "no answer to the request for robots.txt" : `robots.txt answered ${site.robots.status}`;
+	return null;
+}
+async function runCheck(context, parsed, style) {
+	const failOn = failOnOf(parsed);
+	const fetcher = context.createFetcher({ allowPrivateNetwork: flag(parsed, "allow-private-network") });
+	let raw;
+	try {
+		raw = await checkSite(parsed.positionals[0], { fetcher });
+	} finally {
+		await fetcher.close();
+	}
+	const site = sanitizeDeep(raw);
+	const why = unreadable(site);
+	const exit = why !== null ? 3 : exitForFindings(site.findings, failOn);
+	const lines = [style.bold(`AI readiness check: ${site.origin === "" ? parsed.positionals[0] : site.origin}`), ""];
+	if (site.crawlerAccess.length > 0) {
+		lines.push(style.bold("Crawler access (robots.txt, whole site)"));
+		lines.push(renderTable([
+			"Crawler",
+			"Purpose",
+			"Access"
+		], site.crawlerAccess.map((entry) => [
+			entry.token,
+			entry.purpose,
+			entry.access === "partial" ? "partial (some paths closed)" : entry.access
+		]), style));
+		lines.push("");
+	}
+	lines.push(style.bold("Findings"));
+	lines.push(renderFindings(site.findings, style));
+	lines.push("");
+	lines.push(`${renderSummary(site.findings)}`);
+	if (why !== null) {
+		lines.push("", `The site could not be read: ${why}. Exit code 3.`);
+		if (site.robots.status === null && context.offersPrivateNetwork !== false && !flag(parsed, "allow-private-network")) lines.push(PRIVATE_NETWORK_HINT);
+	}
+	lines.push("", site.crawlersCovered, CANNOT_SEE, "", `Full report: ${fullReportUrl("check")}`);
+	return {
+		exit,
+		json: {
+			schemaVersion: 1,
+			command: "check",
+			exitCode: exit,
+			failOn,
+			summary: summarize(site.findings),
+			...why === null ? {} : { error: {
+				code: "unreadable",
+				message: why
+			} },
+			site
+		},
+		text: `${lines.join("\n")}\n`
+	};
+}
+// src/cli/commands/crawlers.ts
+function runCrawlers(style) {
+	return {
+		exit: 0,
+		json: {
+			schemaVersion: 1,
+			command: "crawlers",
+			exitCode: 0,
+			crawlersCovered: crawlersCoveredLine(),
+			crawlers: CRAWLERS.map((crawler) => ({
+				token: crawler.token,
+				vendor: crawler.vendor,
+				purpose: crawler.purpose
+			}))
+		},
+		text: `${renderCrawlerList(style)}\n\n${crawlersCoveredLine()}\n`
+	};
+}
+// src/facts/origin-robots.ts
+/**
+* robots.txt per origin. An address on a subdomain, or on the other scheme, is a different
+* origin with its own robots.txt: before this tool requests anything there, it reads that
+* origin's file once and asks whether its own token may fetch the path. Answers are kept for
+* the run.
+*/
+function createOriginRobots(options) {
+	const known = /* @__PURE__ */ new Map();
+	const robotsOf = (origin) => {
+		const cached = known.get(origin);
+		if (cached !== void 0) return cached;
+		const url = `${origin}/robots.txt`;
+		const read = options.fetcher.fetchText(url, {
+			kind: "robots",
+			...options.signal === void 0 ? {} : { signal: options.signal },
+			...options.redirectAllowed === void 0 ? {} : { redirectAllowed: options.redirectAllowed }
+		}).then((result) => {
+			const { robots } = robotsRead(result, url);
+			if (robots !== null) {
+				const delay = crawlDelayFor(robots, READER_TOKEN);
+				options.fetcher.setCrawlDelay(new URL(origin).hostname, delay === null ? null : Math.min(delay, 10));
+			}
+			return robots;
+		});
+		known.set(origin, read);
+		return read;
+	};
+	return {
+		robotsOf,
+		async check(url) {
+			let parsed;
+			try {
+				parsed = new URL(url);
+			} catch {
+				return "closed";
+			}
+			const robots = await robotsOf(parsed.origin);
+			if (robots === null) return "unreadable";
+			return isAllowedByRobots(robots, "QueryRev-AIReadiness", `${parsed.pathname}${parsed.search}`) ? "allowed" : "closed";
+		}
+	};
+}
+/** Bytes of each page that are read: the part `pageMeta` looks at. */
+var GENERATE_PAGE_BYTES = PAGE_SCAN_MAX_CHARS;
+/** Addresses taken from the home page when the site lists no sitemap. */
+var HOME_LINKS_MAX = 500;
+/** Pages requested per page listed, at most: skipped pages cost requests too. */
+var ATTEMPT_FACTOR = 2;
+var ASSET_EXTENSION = /\.(?:jpe?g|png|gif|svg|webp|avif|ico|pdf|zip|gz|xml|json|css|m?js|txt|mp[34]|webm|woff2?|ttf|otf|docx?|xlsx?|pptx?|csv|rss|atom)$/i;
+/** The addresses of the `<a href>` links of a page, resolved against `base`; at most `max`. */
+function homeLinks(html, base, max = HOME_LINKS_MAX) {
+	const text = html.length > 262144 ? html.slice(0, PAGE_SCAN_MAX_CHARS) : html;
+	const out = [];
+	const seen = /* @__PURE__ */ new Set();
+	let at = 0;
+	while (out.length < max) {
+		const open = text.indexOf("<a", at);
+		if (open === -1) break;
+		const next = text[open + 2];
+		if (next === void 0 || !(next === " " || next === "\n" || next === "	" || next === "\r")) {
+			at = open + 2;
+			continue;
+		}
+		const end = text.indexOf(">", open + 2);
+		if (end === -1) break;
+		at = end + 1;
+		const href = attributes(text.slice(open + 2, end)).get("href");
+		if (href === void 0 || href.trim() === "") continue;
+		let url;
+		try {
+			url = new URL(stripUnsafeCharacters(href.trim()), base);
+		} catch {
+			continue;
+		}
+		if (url.protocol !== "http:" && url.protocol !== "https:") continue;
+		url.hash = "";
+		if (!seen.has(url.href)) {
+			seen.add(url.href);
+			out.push(url.href);
+		}
+	}
+	return out;
+}
+function sitemapText(rawBody, text) {
+	if (rawBody !== null && rawBody.length > 2 && rawBody[0] === 31 && rawBody[1] === 139) try {
+		return gunzipSync(rawBody, { maxOutputLength: FETCH_LIMITS.sitemap }).toString("utf8");
+	} catch {
+		return null;
+	}
+	return text;
+}
+function clampPages(value) {
+	if (value === void 0 || !Number.isFinite(value)) return 30;
+	return Math.min(100, Math.max(1, Math.floor(value)));
+}
+function empty(outcome, origin, host, notes) {
+	return Object.freeze({
+		schemaVersion: 1,
+		outcome,
+		origin,
+		host,
+		text: null,
+		site: null,
+		pages: Object.freeze([]),
+		source: "none",
+		skipped: Object.freeze([]),
+		budgetReached: false,
+		notes: Object.freeze([...notes])
+	});
+}
+async function generateLlmsTxt(input, options = {}) {
+	const registrable = options.registrableDomain ?? registrableDomain;
+	const typed = input.trim();
+	const checked = checkTextUrl(/^[a-z][a-z0-9+.-]*:\/\//i.test(typed) ? typed : `https://${typed}`);
+	if ("refusal" in checked) return empty("refused", "", "", ["The address cannot be checked: it must be a web site name or an http(s) address."]);
+	const origin = checked.url.origin;
+	const host = checked.url.hostname;
+	const siteDomain = registrable(host);
+	const maxPages = clampPages(options.maxPages);
+	const owned = options.fetcher === void 0;
+	const fetcher = options.fetcher ?? createTextFetcher();
+	const budget = AbortSignal.timeout(GENERATE_BUDGET_MS);
+	const signal = options.signal === void 0 ? budget : AbortSignal.any([options.signal, budget]);
+	const sameSite = (target) => registrable(target.hostname) === siteDomain;
+	const notes = [];
+	const skipped = [];
+	try {
+		const origins = createOriginRobots({
+			fetcher,
+			signal,
+			redirectAllowed: sameSite
+		});
+		const robots = await origins.robotsOf(origin);
+		if (robots === null) return empty("robots_unreadable", origin, host, ["robots.txt could not be read, so nothing else was requested."]);
+		const allowed = async (url) => await origins.check(url) === "allowed";
+		let site = {
+			name: host,
+			summary: null
+		};
+		const pages = [];
+		const listed = /* @__PURE__ */ new Set();
+		let homeHtml = null;
+		if (await allowed(`${origin}/`)) {
+			const page = readPage(await fetcher.fetchText(`${origin}/`, {
+				kind: "page",
+				maxBytes: GENERATE_PAGE_BYTES,
+				signal,
+				redirectAllowed: sameSite
+			}), sameSite);
+			if (page.kind === "page") {
+				homeHtml = page.html;
+				const meta = pageMeta(page.html);
+				site = {
+					name: meta.title ?? meta.h1 ?? host,
+					summary: meta.description
+				};
+				if (!meta.noindex && !page.noindex) {
+					pages.push({
+						url: page.finalUrl,
+						title: meta.title ?? meta.h1 ?? host,
+						description: meta.description
+					});
+					listed.add(page.finalUrl);
+				}
+			} else notes.push("The home page could not be read, so the site is named by its host.");
+		} else notes.push("robots.txt keeps this reader from the home page, so it was not requested.");
+		let source = "none";
+		let candidates = [];
+		const fromSitemaps = await sitemapAddresses(fetcher, robots, allowed, sameSite, signal, notes);
+		if (fromSitemaps.length > 0) {
+			source = "sitemap";
+			candidates = fromSitemaps;
+		} else if (homeHtml !== null) {
+			source = "home_links";
+			candidates = homeLinks(homeHtml, `${origin}/`);
+			notes.push("No usable sitemap was found, so the links on the home page were used.");
+		}
+		candidates = candidates.filter((url) => {
+			try {
+				const parsed = new URL(url);
+				return (parsed.protocol === "http:" || parsed.protocol === "https:") && sameSite(parsed) && !ASSET_EXTENSION.test(parsed.pathname);
+			} catch {
+				return false;
+			}
+		});
+		let attempts = 0;
+		const attemptLimit = Math.min(200, maxPages * ATTEMPT_FACTOR);
+		let budgetReached = false;
+		for (const candidate of candidates) {
+			if (pages.length >= maxPages || attempts >= attemptLimit) break;
+			if (signal.aborted) {
+				budgetReached = true;
+				break;
+			}
+			if (listed.has(candidate)) continue;
+			if (!await allowed(candidate)) {
+				skipped.push({
+					url: candidate,
+					reason: "robots"
+				});
+				continue;
+			}
+			attempts += 1;
+			const result = await fetcher.fetchText(candidate, {
+				kind: "page",
+				maxBytes: GENERATE_PAGE_BYTES,
+				signal,
+				redirectAllowed: sameSite
+			});
+			if (result.outcome === "budget_exceeded") {
+				budgetReached = true;
+				break;
+			}
+			const page = readPage(result, sameSite);
+			if (page.kind !== "page") {
+				skipped.push({
+					url: candidate,
+					reason: page.kind
+				});
+				continue;
+			}
+			if (listed.has(page.finalUrl)) {
+				skipped.push({
+					url: candidate,
+					reason: "duplicate"
+				});
+				continue;
+			}
+			const meta = pageMeta(page.html);
+			if (meta.noindex || page.noindex) {
+				skipped.push({
+					url: candidate,
+					reason: "noindex"
+				});
+				continue;
+			}
+			listed.add(page.finalUrl);
+			pages.push({
+				url: page.finalUrl,
+				title: meta.title ?? meta.h1 ?? "",
+				description: meta.description
+			});
+		}
+		if (budgetReached) notes.push("The time limit of 120 seconds was reached; the file lists the pages read so far.");
+		if (pages.length === 0) return Object.freeze({
+			...empty("nothing_found", origin, host, [...notes, "No page could be read from this site."]),
+			skipped: Object.freeze(skipped),
+			budgetReached
+		});
+		return Object.freeze({
+			schemaVersion: 1,
+			outcome: "generated",
+			origin,
+			host,
+			text: renderLlmsTxt(site, pages),
+			site,
+			pages: Object.freeze(pages),
+			source,
+			skipped: Object.freeze(skipped),
+			budgetReached,
+			notes: Object.freeze(notes)
+		});
+	} finally {
+		if (owned) await fetcher.close();
+	}
+}
+function readPage(result, sameSite) {
+	if (result.outcome === "redirect_refused") return { kind: "off_site" };
+	if (result.outcome !== "response" || result.status === null) return { kind: "unreachable" };
+	if (result.status !== 200) return { kind: "status" };
+	if (result.text === null || result.contentType === null || !(result.contentType === "text/html" || result.contentType === "application/xhtml+xml")) return { kind: "not_html" };
+	let final;
+	try {
+		final = new URL(result.finalUrl);
+	} catch {
+		return { kind: "unreachable" };
+	}
+	if (!sameSite(final)) return { kind: "off_site" };
+	final.hash = "";
+	return {
+		kind: "page",
+		html: result.text,
+		finalUrl: final.href,
+		noindex: xRobotsNoindex(result.headers["x-robots-tag"])
+	};
+}
+/** The page addresses of the site's sitemaps; empty when there are none to read. */
+async function sitemapAddresses(fetcher, robots, allowed, sameSite, signal, notes) {
+	const queue = [];
+	for (const line of robots.sitemaps) try {
+		const url = new URL(line);
+		if ((url.protocol === "http:" || url.protocol === "https:") && sameSite(url)) queue.push(url.href);
+	} catch {}
+	const pages = [];
+	const read = /* @__PURE__ */ new Set();
+	let files = 0;
+	while (queue.length > 0 && files < 5 && !signal.aborted) {
+		const url = queue.shift();
+		if (read.has(url)) continue;
+		read.add(url);
+		if (!await allowed(url)) continue;
+		files += 1;
+		const result = await fetcher.fetchText(url, {
+			kind: "sitemap",
+			signal,
+			redirectAllowed: sameSite
+		});
+		if (result.outcome !== "response" || result.status === null || result.status < 200 || result.status >= 300) continue;
+		const body = sitemapText(result.rawBody, result.text);
+		if (body === null) continue;
+		const found = sitemapLocations(body, 5e3);
+		if (found.kind === "index") for (const location of found.locations) try {
+			const child = new URL(location);
+			if (sameSite(child)) queue.push(child.href);
+		} catch {}
+		else if (found.kind === "urlset") pages.push(...found.locations);
+	}
+	if (queue.length > 0 && files >= 5) notes.push(`Only the first 5 sitemap files were read.`);
+	return pages;
+}
+// src/cli/commands/llms-generate.ts
+function pagesOption(value) {
+	if (value === void 0) return 30;
+	if (!/^\d{1,4}$/.test(value)) throw new UsageError(`--max-pages takes a whole number from 1 to 100.`);
+	const count = Number(value);
+	if (count < 1 || count > 100) throw new UsageError(`--max-pages takes a whole number from 1 to 100.`);
+	return count;
+}
+var SOURCE_TEXT = {
+	sitemap: "the sitemap",
+	home_links: "the links on the home page",
+	none: "no source"
+};
+var SKIPPED_TEXT = {
+	robots: "closed to this reader by robots.txt",
+	noindex: "marked noindex",
+	not_html: "not an HTML page",
+	status: "not a 200 answer",
+	off_site: "redirected to another site",
+	unreachable: "no answer",
+	duplicate: "the same page as one listed"
+};
+function skippedSummary(result) {
+	const counts = /* @__PURE__ */ new Map();
+	for (const entry of result.skipped) counts.set(entry.reason, (counts.get(entry.reason) ?? 0) + 1);
+	return [...counts].map(([reason, count]) => `${count} ${SKIPPED_TEXT[reason]}`);
+}
+async function runLlmsGenerate(context, parsed, style) {
+	const maxPages = pagesOption(text(parsed, "max-pages"));
+	const out = text(parsed, "out");
+	const force = flag(parsed, "force");
+	if (out !== void 0) {
+		if (out === "") throw new UsageError("--out needs a file name.");
+		if (!force) {
+			let exists = true;
+			try {
+				await context.readFile(out, 1);
+			} catch (error) {
+				exists = error.code !== "ENOENT";
+			}
+			if (exists) throw new UsageError(`${out.slice(0, 200)} already exists (or cannot be checked). Use --force to replace it.`);
+		}
+	}
+	const fetcher = context.createFetcher({ allowPrivateNetwork: flag(parsed, "allow-private-network") });
+	let raw;
+	try {
+		raw = await generateLlmsTxt(parsed.positionals[0], {
+			maxPages,
+			fetcher
+		});
+	} finally {
+		await fetcher.close();
+	}
+	const result = sanitizeDeep(raw);
+	const exit = result.outcome === "generated" ? 0 : 3;
+	if (result.text !== null && out !== void 0) await context.writeFile(out, result.text, force);
+	const summary = [];
+	if (result.outcome === "generated") {
+		const skipped = skippedSummary(result);
+		summary.push(style.bold(`Generated an llms.txt for ${result.origin}: ${result.pages.length} page${result.pages.length === 1 ? "" : "s"} from ${SOURCE_TEXT[result.source]}.`));
+		if (skipped.length > 0) summary.push(`Left out: ${skipped.join(", ")}.`);
+		if (out !== void 0) summary.push(`Wrote ${out}.`);
+	} else summary.push(style.bold(`No llms.txt was generated for ${result.origin === "" ? parsed.positionals[0] : result.origin}.`));
+	for (const note of result.notes) summary.push(oneLine(note));
+	if (result.outcome === "generated") summary.push("Titles and descriptions are the pages' own words. Read the file and edit it before you publish it.");
+	else {
+		summary.push("Exit code 3.");
+		if (result.outcome === "robots_unreadable" && context.offersPrivateNetwork !== false && !flag(parsed, "allow-private-network")) summary.push("A staging site on a private network is reached with --allow-private-network.");
+	}
+	summary.push("", crawlersCoveredLine(), "", `Full report: ${fullReportUrl("llms-generate")}`);
+	const json = {
+		schemaVersion: 1,
+		command: "llms-generate",
+		exitCode: exit,
+		crawlersCovered: crawlersCoveredLine(),
+		...exit === 3 ? { error: {
+			code: "unreadable",
+			message: result.notes.join(" ")
+		} } : {},
+		...out === void 0 ? {} : { out },
+		result
+	};
+	if (out !== void 0 || result.text === null) return {
+		exit,
+		json,
+		text: `${summary.join("\n")}\n`
+	};
+	return {
+		exit,
+		json,
+		text: result.text,
+		textErr: `${summary.join("\n")}\n`
+	};
+}
+/** Bytes of each linked page that are read. */
+var LLMS_LINK_BODY_BYTES = 65536;
+/** The distinct http(s) addresses among the targets, resolved against `base`, fragments removed. */
+function resolveTargets(targets, base, sameSite) {
+	const seen = /* @__PURE__ */ new Set();
+	const onSite = [];
+	let skipped = 0;
+	for (const target of targets) {
+		let url;
+		try {
+			url = new URL(target, base);
+		} catch {
+			skipped += 1;
+			continue;
+		}
+		url.hash = "";
+		if (url.protocol !== "http:" && url.protocol !== "https:" || !sameSite(url)) {
+			skipped += 1;
+			continue;
+		}
+		if (seen.has(url.href)) continue;
+		seen.add(url.href);
+		onSite.push(url.href);
+	}
+	return {
+		onSite,
+		skipped
+	};
+}
+async function checkLlmsLinks(targets, options) {
+	const registrable = options.registrableDomain ?? registrableDomain;
+	const base = new URL(options.base);
+	const siteDomain = registrable(base.hostname);
+	const { onSite, skipped } = resolveTargets(targets, base, (url) => registrable(url.hostname) === siteDomain);
+	const chosen = onSite.slice(0, 20);
+	const beyondLimit = onSite.length - chosen.length;
+	if (chosen.length === 0) return {
+		skipped,
+		beyondLimit,
+		robotsRead: true,
+		results: []
+	};
+	const budget = AbortSignal.timeout(CHECK_BUDGET_MS);
+	const signal = options.signal === void 0 ? budget : AbortSignal.any([options.signal, budget]);
+	const sameSite = (target) => registrable(target.hostname) === siteDomain;
+	const origins = createOriginRobots({
+		fetcher: options.fetcher,
+		signal,
+		redirectAllowed: sameSite
+	});
+	if (await origins.robotsOf(base.origin) === null) return {
+		skipped,
+		beyondLimit,
+		robotsRead: false,
+		results: []
+	};
+	const results = [];
+	let outOfTime = false;
+	for (const url of chosen) {
+		if (outOfTime || signal.aborted) {
+			outOfTime = true;
+			results.push({
+				url,
+				outcome: "not_checked",
+				status: null
+			});
+			continue;
+		}
+		const access = await origins.check(url);
+		if (signal.aborted) {
+			outOfTime = true;
+			results.push({
+				url,
+				outcome: "not_checked",
+				status: null
+			});
+			continue;
+		}
+		if (access !== "allowed") {
+			results.push({
+				url,
+				outcome: access === "closed" ? "not_allowed" : "robots_unreadable",
+				status: null
+			});
+			continue;
+		}
+		const result = await options.fetcher.fetchText(url, {
+			kind: "page",
+			maxBytes: LLMS_LINK_BODY_BYTES,
+			signal,
+			redirectAllowed: sameSite
+		});
+		if (result.outcome === "budget_exceeded") {
+			outOfTime = true;
+			results.push({
+				url,
+				outcome: "not_checked",
+				status: null
+			});
+			continue;
+		}
+		const status = result.outcome === "response" ? result.status : null;
+		const outcome = status === null ? "unreachable" : status >= 200 && status < 300 ? "ok" : status === 404 || status === 410 ? "missing" : "error_status";
+		results.push({
+			url,
+			outcome,
+			status
+		});
+	}
+	return {
+		skipped,
+		beyondLimit,
+		robotsRead: true,
+		results
+	};
+}
+/** The findings of a link check. One finding for the set; the evidence lists what is wrong. */
+function llmsLinkFindings(report) {
+	if (!report.robotsRead) return [llmsFinding("llms-txt-links", "warn", "robots.txt could not be read, so no link was requested.")];
+	if (report.results.length === 0) return [llmsFinding("llms-txt-links", "info", "The file has no link on this site to check.")];
+	const notRequested = /* @__PURE__ */ new Set([
+		"not_allowed",
+		"robots_unreadable",
+		"not_checked"
+	]);
+	const checked = report.results.filter((entry) => !notRequested.has(entry.outcome));
+	const missing = checked.filter((entry) => entry.outcome === "missing");
+	const other = checked.filter((entry) => entry.outcome === "error_status" || entry.outcome === "unreachable");
+	const notAllowed = report.results.filter((entry) => entry.outcome === "not_allowed" || entry.outcome === "robots_unreadable").length;
+	const notChecked = report.results.filter((entry) => entry.outcome === "not_checked").length;
+	const list = (entries) => entries.slice(0, 5).map((entry) => `${entry.status ?? "no answer"} ${entry.url}`).join("\n");
+	const tail = [
+		notAllowed > 0 ? `${notAllowed} not requested (robots.txt)` : "",
+		notChecked > 0 ? `${notChecked} not checked (time budget)` : "",
+		report.beyondLimit > 0 ? `${report.beyondLimit} beyond the first 20` : ""
+	].filter((part) => part !== "").join("; ");
+	const suffix = tail === "" ? "" : ` (${tail})`;
+	if (checked.length === 0) return [llmsFinding("llms-txt-links", notChecked > 0 ? "warn" : "info", `No link was requested${suffix}.`)];
+	if (missing.length > 0) return [llmsFinding("llms-txt-links", "fail", `${missing.length} of ${checked.length} checked links answer 404 or 410${suffix}.`, list(missing))];
+	if (other.length > 0) return [llmsFinding("llms-txt-links", "warn", `${other.length} of ${checked.length} checked links did not answer with a 2xx${suffix}.`, list(other))];
+	return [llmsFinding("llms-txt-links", "pass", `All ${checked.length} checked link${checked.length === 1 ? "" : "s"} on this site answer 2xx${suffix}.`)];
+}
+// src/cli/commands/llms-validate.ts
+function reportJson(report) {
+	return {
+		present: report.present,
+		verdict: report.verdict,
+		status: report.status,
+		contentType: report.contentType,
+		bytes: report.bytes,
+		truncated: report.truncated,
+		h1: report.h1,
+		h1Count: report.h1Count,
+		structure: report.structure === null ? null : {
+			summary: report.structure.summary,
+			sections: report.structure.sections,
+			links: report.structure.links,
+			structured: report.structure.structured
+		}
+	};
+}
+async function runLlmsValidate(context, parsed, style) {
+	const failOn = failOnOf(parsed);
+	const wantLinks = flag(parsed, "links");
+	const baseOption = text(parsed, "base");
+	let base = null;
+	if (baseOption !== void 0) {
+		const checked = checkTextUrl(baseOption);
+		if ("refusal" in checked) throw new UsageError("--base takes the site address, such as https://example.com.");
+		base = checked.url.origin;
+	}
+	const { source, file } = await classifySource(context, parsed.positionals[0], LLMS_READ_LIMIT);
+	if (source.kind === "file" && wantLinks && base === null) throw new UsageError("--links on a file needs --base <site address> to resolve and limit the links.");
+	let report;
+	let location = source.location;
+	const fetcher = context.createFetcher({ allowPrivateNetwork: flag(parsed, "allow-private-network") });
+	try {
+		if (source.kind === "file" && file !== null) report = validateLlmsTxt({
+			status: 200,
+			contentType: "text/plain",
+			body: decodeFile(file.bytes),
+			truncated: file.truncated
+		});
+		else {
+			const address = fileAddress(source.location, "llms.txt");
+			location = address;
+			const signal = AbortSignal.timeout(CHECK_BUDGET_MS);
+			const result = await fetcher.fetchText(address, {
+				kind: "llms",
+				signal
+			});
+			if (result.outcome !== "response") {
+				const message = `${address} could not be read: ${describeFetchFailure(result, context.offersPrivateNetwork !== false)}`;
+				return {
+					exit: 3,
+					json: sanitizeDeep({
+						schemaVersion: 1,
+						command: "llms-validate",
+						exitCode: 3,
+						source: {
+							kind: "url",
+							location: address
+						},
+						error: {
+							code: "unreadable",
+							message
+						}
+					}),
+					text: `${oneLine(message)}\nExit code 3.\n`
+				};
+			}
+			report = validateLlmsTxt({
+				status: result.status,
+				contentType: result.contentType,
+				body: result.text,
+				truncated: result.truncated
+			});
+			if (base === null) base = new URL(result.finalUrl).origin;
+		}
+		const findings = llmsFindings(report);
+		let links = null;
+		if (wantLinks && report.structure !== null && base !== null) {
+			links = await checkLlmsLinks(report.structure.targets, {
+				fetcher,
+				base,
+				signal: AbortSignal.timeout(CHECK_BUDGET_MS)
+			});
+			findings.push(...llmsLinkFindings(links));
+		}
+		const exit = exitForFindings(findings, failOn);
+		const safe = sanitizeDeep({
+			findings,
+			report: reportJson(report),
+			links,
+			source: {
+				kind: source.kind,
+				location
+			}
+		});
+		const lines = [
+			style.bold(`llms.txt validation: ${safe.source.location}`),
+			"",
+			renderFindings(safe.findings, style)
+		];
+		if (safe.links !== null && safe.links.results.length > 0) lines.push("", style.bold("Links checked"), renderTable([
+			"Result",
+			"Status",
+			"Address"
+		], safe.links.results.map((entry) => [
+			entry.outcome === "ok" ? "OK" : entry.outcome.replace("_", " "),
+			entry.status === null ? "-" : String(entry.status),
+			entry.url
+		]), style));
+		lines.push("", renderSummary(findings), "", crawlersCoveredLine(), "", `Full report: ${fullReportUrl("llms-validate")}`);
+		return {
+			exit,
+			json: {
+				schemaVersion: 1,
+				command: "llms-validate",
+				exitCode: exit,
+				failOn,
+				crawlersCovered: crawlersCoveredLine(),
+				source: safe.source,
+				report: safe.report,
+				...safe.links === null ? {} : { links: safe.links },
+				findings: safe.findings
+			},
+			text: `${lines.join("\n")}\n`
+		};
+	} finally {
+		await fetcher.close();
+	}
+}
+// src/cli/finding.ts
+var DOC_BASE = "https://queryrev.com/research/ai-search-readiness#";
+/** A finding of the command line's own checks (a robots.txt test); the evidence is cut short. */
+function cliFinding(id, level, message, snippet, anchor = "readable-by-ai-search") {
+	const evidence = stripUnsafeCharacters(snippet ?? "").slice(0, 300);
+	const docUrl = `${DOC_BASE}${anchor}`;
+	return Object.freeze(evidence === "" ? {
+		id,
+		level,
+		message: stripUnsafeCharacters(message),
+		docUrl
+	} : {
+		id,
+		level,
+		message: stripUnsafeCharacters(message),
+		docUrl,
+		snippet: evidence
+	});
+}
+// src/cli/commands/robots-test.ts
+var AGENT_TOKEN = /^[A-Za-z0-9._*-]{1,64}$/;
+var PATH_MAX = 2048;
+/** Edit distance, for "did you mean" on a mistyped crawler token. Both strings are short. */
+function distance(a, b) {
+	const row = Array.from({ length: b.length + 1 }, (_, index) => index);
+	for (let i = 1; i <= a.length; i += 1) {
+		let previous = row[0];
+		row[0] = i;
+		for (let j = 1; j <= b.length; j += 1) {
+			const above = row[j];
+			row[j] = Math.min(row[j] + 1, row[j - 1] + 1, previous + (a[i - 1] === b[j - 1] ? 0 : 1));
+			previous = above;
+		}
+	}
+	return row[b.length];
+}
+/** The covered crawler a typed token is probably meant to be, if it is not one. */
+function suggestion(token) {
+	const lower = token.toLowerCase();
+	if (CRAWLERS.some((crawler) => crawler.token.toLowerCase() === lower)) return null;
+	let best = null;
+	for (const crawler of CRAWLERS) {
+		const score = distance(lower, crawler.token.toLowerCase());
+		if (score <= 3 && (best === null || score < best.score)) best = {
+			token: crawler.token,
+			score
+		};
+	}
+	return best?.token ?? null;
+}
+function describeMatch(match) {
+	const group = match.via === "none" ? "no group applies" : match.via === "named" ? `the group for this agent (line ${match.group?.line ?? 0})` : `the "*" group (line ${match.group?.line ?? 0})`;
+	if (match.rule === null) return {
+		sentence: match.via === "none" ? "no group applies, so nothing is closed" : `${group}, and no rule matches`,
+		evidence: match.group === null ? "no group applies" : `${match.via === "named" ? "group for this agent" : "\"*\" group"} at line ${match.group.line}, no rule matches`
+	};
+	const rule = `${match.rule.allow ? "Allow" : "Disallow"}: ${match.rule.pattern}`;
+	return {
+		sentence: `${group}, decided by line ${match.rule.line} (${rule})`,
+		evidence: `line ${match.rule.line}: ${rule}`
+	};
+}
+function flatMatch(match) {
+	return {
+		allowed: match.allowed,
+		via: match.via,
+		group: match.group === null ? null : {
+			line: match.group.line,
+			agents: [...match.group.agents]
+		},
+		rule: match.rule === null ? null : {
+			type: match.rule.allow ? "allow" : "disallow",
+			pattern: match.rule.pattern,
+			line: match.rule.line
+		}
+	};
+}
+async function load(context, parsed) {
+	const { source, file } = await classifySource(context, parsed.positionals[0], ROBOTS_READ_LIMIT);
+	const notes = [];
+	if (source.kind === "file" && file !== null) {
+		const body = decodeFile(file.bytes);
+		if (looksLikeHtml(null, body)) throw new UsageError(`${source.location.slice(0, 200)} is an HTML page, not a robots.txt.`);
+		if (file.truncated) notes.push(cliFinding("robots-file-cut", "info", "The file is larger than 500 KiB; it was read as far as that limit."));
+		return {
+			robots: parseRobots(body),
+			source,
+			notes
+		};
+	}
+	const address = fileAddress(source.location, "robots.txt");
+	const fetcher = context.createFetcher({ allowPrivateNetwork: flag(parsed, "allow-private-network") });
+	try {
+		const result = await fetcher.fetchText(address, {
+			kind: "robots",
+			signal: AbortSignal.timeout(CHECK_BUDGET_MS)
+		});
+		const { read, robots } = robotsRead(result, address);
+		if (robots === null) return {
+			unreadable: `${address}: ${describeFetchFailure(result, context.offersPrivateNetwork !== false)}`,
+			source
+		};
+		if (read.outcome === "missing") notes.push(cliFinding("robots-missing", "info", `There is no robots.txt at ${address} (status ${read.status ?? 0}), so nothing is closed.`));
+		if (read.truncated) notes.push(cliFinding("robots-file-cut", "info", "The file is larger than 500 KiB; it was read as far as that limit."));
+		return {
+			robots,
+			source: {
+				kind: "url",
+				location: address
+			},
+			notes
+		};
+	} finally {
+		await fetcher.close();
+	}
+}
+async function runRobotsTest(context, parsed, style) {
+	const failOn = failOnOf(parsed);
+	const agent = text(parsed, "agent");
+	const path = text(parsed, "path") ?? "/";
+	if (agent !== void 0 && !AGENT_TOKEN.test(agent)) throw new UsageError("--agent takes a crawler token such as OAI-SearchBot (letters, digits, . _ - *).");
+	if (!path.startsWith("/") || path.length > PATH_MAX || [...path].some((char) => char <= " " || char === "")) throw new UsageError("--path must start with \"/\" and hold no spaces or control characters.");
+	const loaded = await load(context, parsed);
+	if ("unreadable" in loaded) {
+		const message = `robots.txt could not be read: ${loaded.unreadable}`;
+		const exit = 3;
+		return {
+			exit,
+			json: sanitizeDeep({
+				schemaVersion: 1,
+				command: "robots-test",
+				exitCode: exit,
+				source: loaded.source,
+				error: {
+					code: "unreadable",
+					message
+				}
+			}),
+			text: `${oneLine(message)}\nExit code 3.\n`
+		};
+	}
+	const { robots, source, notes } = loaded;
+	const findings = [...notes];
+	if (robots.groups.length === 0) findings.push(cliFinding("robots-groups", "info", "The file has no User-agent group, so nothing is closed."));
+	const lines = [
+		style.bold(`robots.txt test: ${source.location}`),
+		`Path: ${path}`,
+		""
+	];
+	let detail;
+	let covered = null;
+	if (agent !== void 0) {
+		const match = matchRobots(robots, agent, path);
+		const { sentence, evidence } = describeMatch(match);
+		const level = match.allowed ? "pass" : "fail";
+		findings.push(cliFinding("robots-access", level, `${agent} ${match.allowed ? "may" : "may not"} fetch ${path}: ${sentence}.`, evidence));
+		const near = suggestion(agent);
+		if (near !== null) findings.push(cliFinding("robots-agent-unlisted", "info", `${agent} is not one of the crawlers this tool covers. Did you mean ${near}? The rules are tested for any token you type.`));
+		else if (!CRAWLERS.some((crawler) => crawler.token.toLowerCase() === agent.toLowerCase())) findings.push(cliFinding("robots-agent-unlisted", "info", `${agent} is not one of the crawlers this tool covers; the rules are tested for any token you type.`));
+		lines.push(`Agent: ${agent}`, "", renderFindings(findings, style));
+		detail = {
+			agent,
+			match: flatMatch(match)
+		};
+	} else {
+		const rows = [];
+		const entries = crawlerAccess(robots, { path }).map((entry) => {
+			const match = entry.path ?? entry.root;
+			const training = entry.crawler.purpose === "training";
+			const level = !match.allowed ? training ? "info" : "fail" : !training && path === "/" && entry.access === "partial" ? "warn" : "pass";
+			const { sentence, evidence } = describeMatch(match);
+			findings.push(cliFinding(`robots-access-${entry.crawler.token.toLowerCase()}`, level, `${entry.crawler.token} ${match.allowed ? "may" : "may not"} fetch ${path}: ${sentence}.${level === "warn" ? " Parts of the site are closed to it." : ""}`, evidence));
+			rows.push([
+				level.toUpperCase(),
+				entry.crawler.token,
+				entry.crawler.purpose,
+				match.allowed ? "allowed" : "blocked",
+				evidence
+			]);
+			return {
+				token: entry.crawler.token,
+				purpose: entry.crawler.purpose,
+				vendor: entry.crawler.vendor,
+				siteAccess: entry.access,
+				match: flatMatch(match)
+			};
+		});
+		lines.push(renderTable([
+			"Level",
+			"Crawler",
+			"Purpose",
+			"Access",
+			"Decided by"
+		], rows, style));
+		const other = findings.filter((finding) => !finding.id.startsWith("robots-access-"));
+		if (other.length > 0) lines.push("", renderFindings(other, style));
+		detail = {
+			crawlersCovered: crawlersCoveredLine(),
+			crawlers: entries
+		};
+		covered = crawlersCoveredLine();
+	}
+	const exit = exitForFindings(findings, failOn);
+	const safe = sanitizeDeep({
+		findings,
+		detail,
+		source
+	});
+	lines.push("", renderSummary(findings));
+	if (covered !== null) lines.push("", covered);
+	lines.push("", `Full report: ${fullReportUrl("robots-test")}`);
+	return {
+		exit,
+		json: {
+			schemaVersion: 1,
+			command: "robots-test",
+			exitCode: exit,
+			failOn,
+			path,
+			source: safe.source,
+			...safe.detail,
+			findings: safe.findings
+		},
+		text: `${lines.join("\n")}\n`
+	};
+}
+// src/cli/usage.ts
+var USAGE = `ai-readiness: checks how a site's public files treat AI crawlers.
+
+Usage: ai-readiness <command> [arguments] [options]
+
+Commands
+  check <url>                       robots.txt access of the covered AI crawlers, llms.txt, the
+                                    home page as served, structured data and the sitemap
+  robots test <file|url>            which robots.txt line decides for a crawler and a path
+      --agent <token>                 one crawler token (any token); all covered crawlers without it
+      --path <path>                   the path to test, default /
+  llms validate <file|url>          checks an llms.txt against llmstxt.org
+      --links                         also request up to 20 links on the same site
+      --base <url>                    the site of a local file's links (needed with --links)
+  llms generate <url>               builds the skeleton of an llms.txt from the site's own pages
+      --max-pages <n>                 pages to list, 1 to 100, default 30
+      --out <file>                    write the file here (--force replaces an existing file)
+  crawlers                          the AI crawlers covered, with vendor and purpose
+
+Options
+  --json                 one JSON document with schemaVersion 1, no colour, no other output
+  --fail-on <level>      exit 1 for findings at "fail" (default) or at "warn" (check, robots test, llms validate)
+  --allow-private-network
+                         reach a staging site on a private network (never cloud metadata addresses)
+  --no-color             no colour (also off without a terminal, or when NO_COLOR is set)
+  -h, --help             this text
+  -v, --version          the version
+
+Exit codes
+  0  nothing at or above --fail-on
+  1  checks failed
+  2  usage error
+  3  the site could not be read (refused, unreachable, timed out)
+  4  internal error
+
+Reads public files only. Sends nothing to queryrev.com.
+`;
+// src/cli/main.ts
+var CLI_VERSION = {
+	name: "@queryrev/ai-readiness",
+	version: "1.0.0",
+	description: "robots.txt parsing, AI crawler access verdicts, llms.txt checks and a guarded network layer. No model calls.",
+	license: "MIT",
+	author: "QueryRev",
+	homepage: "https://queryrev.com/research/ai-search-readiness",
+	repository: {
+		"type": "git",
+		"url": "git+https://github.com/QueryRevAI/ai-readiness.git"
+	},
+	bugs: { "url": "https://github.com/QueryRevAI/ai-readiness/issues" },
+	keywords: [
+		"robots.txt",
+		"llms.txt",
+		"ai-crawlers",
+		"geo",
+		"aeo",
+		"seo",
+		"cli"
+	],
+	type: "module",
+	engines: { "node": ">=22.19" },
+	sideEffects: false,
+	main: "./dist/index.js",
+	types: "./dist/index.d.ts",
+	bin: { "ai-readiness": "./dist/cli.js" },
+	exports: {
+		".": {
+			"types": "./dist/index.d.ts",
+			"default": "./dist/index.js"
+		},
+		"./cli": {
+			"types": "./dist/net/cli.d.ts",
+			"default": "./dist/net-cli.js"
+		},
+		"./run": {
+			"types": "./dist/cli/run.d.ts",
+			"default": "./dist/run.js"
+		},
+		"./testing": {
+			"types": "./dist/net/testing/index.d.ts",
+			"default": "./dist/testing.js"
+		},
+		"./package.json": "./package.json"
+	},
+	files: [
+		"dist",
+		"data",
+		"README.md",
+		"LICENSE",
+		"CITATION.cff",
+		"SECURITY.md"
+	],
+	publishConfig: { "access": "public" },
+	scripts: {
+		"build": "vite build --config vite.build.config.ts && node scripts/build-types.mjs",
+		"prepublishOnly": "node scripts/check-publishable.mjs && npm run build",
+		"typecheck": "tsc --noEmit",
+		"test": "vitest run"
+	},
+	dependencies: { "undici": "^8.12.3" },
+	devDependencies: {
+		"@types/node": "^24.0.0",
+		"eslint": "^9.39.5",
+		"typescript": "~6.0.3",
+		"vite": "^8.3.4",
+		"vitest": "^5.0.3"
+	}
+}.version;
+async function dispatch(context, parsed, style) {
+	switch (parsed.command) {
+		case "check": return runCheck(context, parsed, style);
+		case "robots test": return runRobotsTest(context, parsed, style);
+		case "llms validate": return runLlmsValidate(context, parsed, style);
+		case "llms generate": return runLlmsGenerate(context, parsed, style);
+		case "crawlers": return runCrawlers(style);
+	}
+}
+function errorDocument(code, message, exit) {
+	return `${JSON.stringify({
+		schemaVersion: 1,
+		exitCode: exit,
+		error: {
+			code,
+			message
+		}
+	}, null, 2)}\n`;
+}
+/**
+* Runs the command line. Returns the exit code and sets no global state: output goes through the
+* context, so a test can run it with a fixture fetcher and read what it printed.
+*/
+async function runCli(argv, context) {
+	const json = argv.includes("--json");
+	try {
+		if (argv.length === 0) {
+			context.stderr.write(USAGE);
+			return 2;
+		}
+		if (argv[0] === "--help" || argv[0] === "-h" || argv[0] === "help") {
+			context.stdout.write(USAGE);
+			return 0;
+		}
+		if (argv[0] === "--version" || argv[0] === "-v") {
+			context.stdout.write(`${CLI_VERSION}\n`);
+			return 0;
+		}
+		const parsed = parseCommand(argv, context.offersPrivateNetwork !== false);
+		if (flag(parsed, "help")) {
+			context.stdout.write(USAGE);
+			return 0;
+		}
+		const result = await dispatch(context, parsed, makeStyle({
+			isTTY: context.stdout.isTTY === true && !flag(parsed, "json"),
+			env: context.env,
+			noColorFlag: flag(parsed, "no-color")
+		}));
+		if (flag(parsed, "json")) context.stdout.write(`${JSON.stringify(result.json, null, 2)}\n`);
+		else {
+			context.stdout.write(result.text);
+			if (result.textErr !== void 0) context.stderr.write(result.textErr);
+		}
+		return result.exit;
+	} catch (error) {
+		if (error instanceof UsageError) {
+			const message = oneLine(error.message);
+			if (json) context.stdout.write(errorDocument("usage", message, 2));
+			else context.stderr.write(`ai-readiness: ${message}\nRun "ai-readiness --help" for usage.\n`);
+			return 2;
+		}
+		const message = `Internal error: ${error instanceof Error ? oneLine(error.name) : "unknown"}.`;
+		if (json) context.stdout.write(errorDocument("internal", message, 4));
+		else context.stderr.write(`ai-readiness: ${message}\n`);
+		return 4;
+	}
+}
+export { runCli as n, CLI_VERSION as t };