@coywolf/emdash 0.0.0-stage → 0.35.13

package/src/robots/explain.ts

@@ -0,0 +1,191 @@
+/**
+ * Plain-English descriptions of rules and of the whole robots.txt, for
+ * people who don't read robots.txt syntax.
+ */
+import { evaluateParsed, parse } from "./rep.js";
+import {
+	CRAWLER_GROUPS,
+	type DirectoryBot,
+	PROBE_AGENT,
+	type RobotsConfig,
+	type RobotsRule,
+	generate,
+	groupTokens,
+	matchGroup,
+	resolveAgents,
+	ruleExts,
+	ruleParams,
+} from "./rules.js";
+
+const FILE_TYPE_NAMES: Record<string, string> = {
+	pdf: "PDF",
+	doc: "Word",
+	docx: "Word",
+	xls: "Excel",
+	xlsx: "Excel",
+	csv: "CSV",
+	ppt: "PowerPoint",
+	pptx: "PowerPoint",
+	zip: "ZIP",
+	jpg: "JPEG",
+	jpeg: "JPEG",
+	png: "PNG",
+	gif: "GIF",
+	webp: "WebP",
+	avif: "AVIF",
+	svg: "SVG",
+	mp4: "MP4",
+	mov: "QuickTime",
+	webm: "WebM",
+	mp3: "MP3",
+};
+
+export function listPhrase(items: string[], max = 3): string {
+	if (items.length <= 1) return items[0] ?? "";
+	if (items.length <= max) return `${items.slice(0, -1).join(", ")} and ${items[items.length - 1]}`;
+	return `${items.slice(0, max).join(", ")} and ${items.length - max} more`;
+}
+
+/** "all crawlers", "AI training crawlers", "GPTBot and ClaudeBot"… */
+export function describeAgents(rule: Pick<RobotsRule, "agents" | "group">, nameOf?: (token: string) => string): string {
+	if (rule.agents.includes("*")) {
+		const others = rule.agents.filter((a) => a !== "*");
+		return others.length ? `all crawlers (including ${listPhrase(others)})` : "all crawlers";
+	}
+	const group = (rule.group && CRAWLER_GROUPS.find((g) => g.id === rule.group)) || matchGroup(rule.agents);
+	if (group) {
+		const inGroup = new Set(group.tokens.map((t) => t.toLowerCase()));
+		const extra = rule.agents.filter((a) => !inGroup.has(a.toLowerCase()));
+		const count = rule.agents.length;
+		if (!extra.length || rule.group === group.id) return `${group.phrase} (${count})`;
+		return `${group.phrase} and ${listPhrase(extra.map((a) => nameOf?.(a) ?? a), 2)}`;
+	}
+	return listPhrase(rule.agents.map((a) => nameOf?.(a) ?? a));
+}
+
+function extsPhrase(exts: string[]): string {
+	const names = [...new Set(exts.map((e) => FILE_TYPE_NAMES[e] ?? `.${e}`))];
+	return `${listPhrase(names, 4)} files`;
+}
+
+/** What part of the site a rule covers, e.g. "the /private/ section and everything in it". */
+export function describeTarget(rule: RobotsRule): string {
+	const path = rule.path ?? "";
+	switch (rule.kind) {
+		case "entire_site":
+			return "the whole site";
+		case "folder": {
+			const p = path.endsWith("/") ? path : `${path}/`;
+			const folder = p.startsWith("/") ? p : `/${p}`;
+			return rule.anyDepth
+				? `every ${folder.replace(/^\/|\/$/g, "") || "/"}/ folder, at any level, and everything in them`
+				: `the ${folder} section and everything in it`;
+		}
+		case "prefix":
+			return `every address starting with ${path.startsWith("/") ? path : `/${path}`}`;
+		case "single_page":
+			return rule.strict ? `only the exact address ${path}` : `the page ${path} (and addresses that continue it)`;
+		case "exact_url":
+			return `only the exact address ${path.startsWith("/") ? path : `/${path}`}`;
+		case "filetype":
+			return `${extsPhrase(ruleExts(rule))} anywhere on the site`;
+		case "filetype_in_folder":
+			return `${extsPhrase(ruleExts(rule))} in ${path.endsWith("/") ? path : `${path}/`}`;
+		case "contains":
+			return `addresses containing “${path}”`;
+		case "any_depth":
+			return `any ${path.replace(/^\/+|\/+$/g, "")}/ folder below the top level`;
+		case "query_any":
+			return "addresses with a query string (anything after a ?)";
+		case "query_param": {
+			const params = ruleParams(rule).map((p) => (p.endsWith("*") ? `${p.slice(0, -1)}…` : p));
+			return `addresses with the ${listPhrase(params, 4)} parameter${params.length > 1 ? "s" : ""}`;
+		}
+		case "wildcard_prefix":
+			return `addresses matching ${path.endsWith("*") ? path : `${path}*`}`;
+		case "allow_exception":
+			return `the ${path} section, except ${rule.allow ?? ""}`;
+		default:
+			return path ? `addresses matching ${path}` : "nothing (an empty value)";
+	}
+}
+
+/** One sentence: "Block AI training crawlers (11) from the whole site". */
+export function describeRule(rule: RobotsRule, nameOf?: (token: string) => string): string {
+	const who = describeAgents(rule, nameOf);
+	const what = describeTarget(rule);
+	let sentence =
+		rule.kind === "allow_exception"
+			? `Block ${who} from ${what}`
+			: rule.directive === "allow"
+				? `Let ${who} crawl ${what}`
+				: `Block ${who} from ${what}`;
+	if (rule.directive === "disallow" && rule.except?.length) sentence += `, except ${listPhrase(rule.except, 3)}`;
+	return sentence;
+}
+
+export interface SummaryLine {
+	who: string;
+	text: string;
+	tone: "open" | "partial" | "closed";
+}
+
+/**
+ * A plain-English summary of what the whole file does, by kind of crawler:
+ * "Search engines can crawl everything except EmDash's admin."
+ */
+export function summarize(config: RobotsConfig, siteUrl: string, bots: DirectoryBot[] = []): SummaryLine[] {
+	const text = generate(config, { siteUrl });
+	const parsed = parse(text).directives;
+	const resolved = resolveAgents(config);
+	const ruleById = new Map(config.rules.map((r) => [r.id, r]));
+	const out: SummaryLine[] = [];
+	const describeFor = (tokens: string[], who: string) => {
+		const states = tokens.map((t) => {
+			const home = evaluateParsed(parsed, [t], "/").allowed;
+			const page = evaluateParsed(parsed, [t], "/some-page/").allowed;
+			const lines = resolved.get(t.toLowerCase())?.lines ?? resolved.get("*")?.lines ?? [];
+			const blocks = lines.filter((l) => !l.system && l.directive === "Disallow");
+			const allows = lines.filter((l) => !l.system && l.directive === "Allow");
+			const areas = (list: typeof lines) =>
+				[...new Set(list.flatMap((l) => l.ruleIds.map((id) => ruleById.get(id))).filter(Boolean).map((r) => describeTarget(r as RobotsRule)))];
+			if (!home && !page) {
+				const open = areas(allows);
+				return open.length ? { tone: "partial" as const, text: `are blocked from the whole site except ${listPhrase(open, 3)}` } : { tone: "closed" as const, text: "are blocked from the whole site" };
+			}
+			const closed = areas(blocks);
+			if (closed.length) return { tone: "partial" as const, text: `can crawl everything except ${listPhrase(closed, 3)}` };
+			return { tone: "open" as const, text: "can crawl everything" };
+		});
+		if (!states.length) return;
+		const buckets = new Map<string, { tone: SummaryLine["tone"]; tokens: string[] }>();
+		states.forEach((st, i) => {
+			const b = buckets.get(st.text) ?? { tone: st.tone, tokens: [] };
+			b.tokens.push(tokens[i]);
+			buckets.set(st.text, b);
+		});
+		if (buckets.size === 1) {
+			const [[text, b]] = [...buckets];
+			out.push({ who, text: tokens.length === 1 && who !== "All other crawlers" ? text.replace(/^are /, "is ") : text, tone: b.tone });
+			return;
+		}
+		const sorted = [...buckets].sort((a, b) => a[1].tokens.length - b[1].tokens.length);
+		const parts = sorted.map(([text, b], i) => (i === sorted.length - 1 && b.tokens.length > 3 ? `the others ${text}` : `${listPhrase(b.tokens, 3)} ${b.tokens.length === 1 ? text.replace(/^are /, "is ") : text}`));
+		out.push({ who: `${who}:`, text: parts.join("; "), tone: "partial" });
+	};
+	const named = new Set<string>();
+	for (const group of CRAWLER_GROUPS) {
+		const tokens = groupTokens(group, bots);
+		for (const t of tokens) named.add(t.toLowerCase());
+		describeFor(tokens, group.phrase.charAt(0).toUpperCase() + group.phrase.slice(1));
+	}
+	const others = [...resolved.values()].map((e) => e.token).filter((t) => t !== "*" && !named.has(t.toLowerCase()));
+	if (others.length) describeFor(others, listPhrase(others, 3));
+	describeFor([PROBE_AGENT], "All other crawlers");
+	return out;
+}
+
+/** "Search engines can crawl everything." as one string per line. */
+export function summaryText(lines: SummaryLine[]): string[] {
+	return lines.map((l) => `${l.who} ${l.text}.`);
+}

package/src/robots/guided.ts

@@ -0,0 +1,300 @@
+/**
+ * The guided "Add rule" flow's model: a small set of plain choices that
+ * map onto stored rules (and back, for editing), smart reading of pasted
+ * addresses, and live "will match / won't match" examples from the real
+ * matcher.
+ *
+ * Choices → stored kinds:
+ *   Everything                        → entire_site
+ *   A section (+ "at any depth")      → folder (+ anyDepth)
+ *   One page (+ "only this address")  → single_page / exact_url
+ *   A kind of file (+ "only in…")     → filetype / filetype_in_folder
+ *   Links with parameters             → query_param / query_any
+ *   Advanced pattern                  → custom (covers prefix, contains, wildcard…)
+ *   Block … "Except…"                 → except[] (more specific Allow lines)
+ */
+import { matchRaw } from "./rep.js";
+import {
+	CRAWLER_GROUPS,
+	type DirectoryBot,
+	type RobotsRule,
+	directives,
+	groupTokens,
+	matchGroup,
+	normalizePathInput,
+	ruleExts,
+	ruleParams,
+	samplePaths,
+} from "./rules.js";
+
+export type Area = "everything" | "section" | "page" | "files" | "params" | "advanced";
+export type Who = "everyone" | "search_engines" | "ai_training" | "ai_search" | "seo_tools" | "specific";
+
+export interface Draft {
+	id: string;
+	enabled: boolean;
+	action: "block" | "allow";
+	area: Area;
+	section: string;
+	anyDepth: boolean;
+	page: string;
+	exact: boolean;
+	fileTypes: string[];
+	otherExt: string;
+	filesIn: string;
+	params: string[];
+	anyQuery: boolean;
+	otherParam: string;
+	pattern: string;
+	except: string[];
+	who: Who;
+	agents: string[];
+	name: string;
+	description: string;
+	nameTouched: boolean;
+	descriptionTouched: boolean;
+}
+
+export const FILE_TYPES: Array<{ id: string; label: string; exts: string[] }> = [
+	{ id: "pdf", label: "PDF", exts: ["pdf"] },
+	{ id: "word", label: "Word", exts: ["doc", "docx"] },
+	{ id: "sheets", label: "Spreadsheets", exts: ["xls", "xlsx", "csv"] },
+	{ id: "slides", label: "Presentations", exts: ["ppt", "pptx"] },
+	{ id: "images", label: "Images", exts: ["jpg", "jpeg", "png", "gif", "webp", "avif", "svg"] },
+	{ id: "zip", label: "ZIP archives", exts: ["zip"] },
+	{ id: "video", label: "Video", exts: ["mp4", "mov", "webm"] },
+	{ id: "audio", label: "Audio", exts: ["mp3", "m4a", "wav"] },
+];
+
+export const PARAMS: Array<{ id: string; label: string; names: string[]; hint: string }> = [
+	{ id: "utm", label: "utm_… (campaign tracking)", names: ["utm_*"], hint: "?utm_source=newsletter" },
+	{ id: "ref", label: "ref", names: ["ref"], hint: "?ref=twitter" },
+	{ id: "fbclid", label: "fbclid (Facebook)", names: ["fbclid"], hint: "?fbclid=…" },
+	{ id: "gclid", label: "gclid (Google Ads)", names: ["gclid"], hint: "?gclid=…" },
+	{ id: "session", label: "Session IDs", names: ["sessionid", "sid"], hint: "?sessionid=…" },
+	{ id: "sort", label: "Sorting and filters", names: ["sort", "order", "filter"], hint: "?sort=price" },
+	{ id: "search", label: "Site search", names: ["q", "s", "search"], hint: "?q=oats" },
+];
+
+export function blankDraft(id: string): Draft {
+	return {
+		id,
+		enabled: true,
+		action: "block",
+		area: "section",
+		section: "",
+		anyDepth: false,
+		page: "",
+		exact: false,
+		fileTypes: [],
+		otherExt: "",
+		filesIn: "",
+		params: [],
+		anyQuery: false,
+		otherParam: "",
+		pattern: "",
+		except: [],
+		who: "everyone",
+		agents: [],
+		name: "",
+		description: "",
+		nameTouched: false,
+		descriptionTouched: false,
+	};
+}
+
+const splitList = (s: string) =>
+	s
+		.split(/[\s,]+/)
+		.map((x) => x.trim())
+		.filter(Boolean);
+
+export function draftExts(d: Draft): string[] {
+	return [...new Set([...FILE_TYPES.filter((f) => d.fileTypes.includes(f.id)).flatMap((f) => f.exts), ...splitList(d.otherExt).map((e) => e.replace(/^\.+/, "").toLowerCase())])];
+}
+
+export function draftParams(d: Draft): string[] {
+	return [...new Set([...PARAMS.filter((p) => d.params.includes(p.id)).flatMap((p) => p.names), ...splitList(d.otherParam).map((p) => p.replace(/^[?&]+/, "").replace(/=.*$/, ""))])];
+}
+
+export function draftAgents(d: Draft, bots: DirectoryBot[]): string[] {
+	if (d.who === "everyone") return ["*"];
+	if (d.who === "specific") return d.agents;
+	const group = CRAWLER_GROUPS.find((g) => g.id === d.who);
+	return group ? groupTokens(group, bots) : [];
+}
+
+const asFolder = (v: string) => (v === "" ? "" : v.endsWith("/") ? v : `${v}/`);
+
+/** The stored rule a draft produces (name/description filled by the caller). */
+export function draftToRule(d: Draft, bots: DirectoryBot[], siteUrl = ""): RobotsRule {
+	const norm = (v: string) => normalizePathInput(v, siteUrl).value;
+	const base = {
+		id: d.id,
+		name: d.name,
+		description: d.description || undefined,
+		enabled: d.enabled,
+		agents: draftAgents(d, bots),
+		directive: d.action === "allow" ? ("allow" as const) : ("disallow" as const),
+		...(d.who !== "everyone" && d.who !== "specific" ? { group: d.who } : {}),
+		source: "user" as const,
+	};
+	const except = d.action === "block" && (d.area === "section" || d.area === "everything" || d.area === "advanced") ? d.except.map((e) => e.trim()).filter(Boolean).map(norm) : [];
+	const withExcept = except.length ? { except } : {};
+	switch (d.area) {
+		case "everything":
+			return { ...base, kind: "entire_site", ...withExcept };
+		case "section":
+			return { ...base, kind: "folder", path: asFolder(norm(d.section)), ...(d.anyDepth ? { anyDepth: true } : {}), ...withExcept };
+		case "page":
+			return { ...base, kind: d.exact ? "exact_url" : "single_page", path: norm(d.page).replace(/\$$/, "") };
+		case "files": {
+			const exts = draftExts(d);
+			const folder = d.filesIn.trim() ? asFolder(norm(d.filesIn)) : "";
+			return folder ? { ...base, kind: "filetype_in_folder", path: folder, exts } : { ...base, kind: "filetype", exts };
+		}
+		case "params":
+			return d.anyQuery ? { ...base, kind: "query_any" } : { ...base, kind: "query_param", params: draftParams(d) };
+		default:
+			return { ...base, kind: "custom", path: d.pattern.trim() ? norm(d.pattern) : "", ...withExcept };
+	}
+}
+
+/** A draft that edits an existing rule (anything the plain choices can't show opens as Advanced). */
+export function ruleToDraft(rule: RobotsRule): Draft {
+	const d = blankDraft(rule.id);
+	d.enabled = rule.enabled;
+	d.action = rule.directive === "allow" && rule.kind !== "allow_exception" ? "allow" : "block";
+	d.name = rule.name;
+	d.description = rule.description ?? "";
+	d.nameTouched = true;
+	d.descriptionTouched = Boolean(rule.description);
+	d.except = [...(rule.except ?? [])];
+	const path = rule.path ?? "";
+	switch (rule.kind) {
+		case "entire_site":
+			d.area = "everything";
+			break;
+		case "folder":
+			d.area = "section";
+			d.section = path;
+			d.anyDepth = Boolean(rule.anyDepth);
+			break;
+		case "allow_exception":
+			d.area = "section";
+			d.section = path;
+			if (rule.allow) d.except = [rule.allow, ...d.except];
+			break;
+		case "single_page":
+			d.area = "page";
+			d.page = path;
+			d.exact = Boolean(rule.strict);
+			break;
+		case "exact_url":
+			d.area = "page";
+			d.page = path;
+			d.exact = true;
+			break;
+		case "filetype":
+		case "filetype_in_folder": {
+			d.area = "files";
+			const exts = ruleExts(rule);
+			const chips = FILE_TYPES.filter((f) => f.exts.every((e) => exts.includes(e)));
+			const covered = new Set(chips.flatMap((c) => c.exts));
+			d.fileTypes = chips.map((c) => c.id);
+			d.otherExt = exts.filter((e) => !covered.has(e)).join(", ");
+			d.filesIn = rule.kind === "filetype_in_folder" ? path : "";
+			break;
+		}
+		case "query_any":
+			d.area = "params";
+			d.anyQuery = true;
+			break;
+		case "query_param": {
+			d.area = "params";
+			const names = ruleParams(rule);
+			const chips = PARAMS.filter((p) => p.names.every((n) => names.includes(n)));
+			const covered = new Set(chips.flatMap((c) => c.names));
+			d.params = chips.map((c) => c.id);
+			d.otherParam = names.filter((n) => !covered.has(n)).join(", ");
+			break;
+		}
+		default: {
+			d.area = "advanced";
+			const lines = directives({ ...rule, except: [] });
+			d.pattern = lines[0]?.value ?? path;
+		}
+	}
+	if (rule.agents.length === 1 && rule.agents[0] === "*") d.who = "everyone";
+	else {
+		const group = (rule.group && CRAWLER_GROUPS.find((g) => g.id === rule.group)) || matchGroup(rule.agents);
+		if (group && rule.group === group.id) d.who = group.id as Who;
+		else {
+			d.who = "specific";
+			d.agents = [...rule.agents];
+		}
+	}
+	return d;
+}
+
+export interface Reading {
+	area: Area;
+	value: string;
+	/** "We read this as: the /recipes/ section" */
+	message: string;
+	notes: string[];
+}
+
+/** Read a pasted address or path and suggest the plain choice that fits it. */
+export function inferInput(input: string, siteUrl = "", sections: string[] = []): Reading | null {
+	const raw = input.trim();
+	if (!raw) return null;
+	const { value, notes } = normalizePathInput(raw, siteUrl);
+	if (/[*$]/.test(value)) return { area: "advanced", value, message: `“${value}” is a pattern (* means anything, $ means the address ends there), so it's set up as an advanced pattern.`, notes };
+	if (value === "/") return { area: "everything", value, message: "That's your home page address, which covers the whole site.", notes };
+	const q = value.indexOf("?");
+	if (q !== -1) {
+		const name = value.slice(q + 1).split(/[=&]/)[0];
+		if (name) return { area: "params", value: name, message: `That address has a ?${name}= parameter. Do you mean links with that parameter?`, notes };
+	}
+	const ext = /\/[^/]+\.([A-Za-z0-9]{1,8})$/.exec(value)?.[1];
+	if (ext && !/^html?$/i.test(ext)) return { area: "page", value, message: `We read this as one ${ext.toUpperCase()} file. To cover every .${ext.toLowerCase()} file, pick “A kind of file”.`, notes };
+	const known = sections.find((s) => value === s || `${value}/` === s);
+	if (value.endsWith("/") || known) {
+		const folder = known ?? value;
+		return { area: "section", value: folder, message: `We read this as the ${folder} section and everything in it.`, notes };
+	}
+	return { area: "page", value, message: `We read this as the page ${value}.`, notes };
+}
+
+/** Example addresses a rule matches and nearby ones it doesn't, from the real matcher. */
+export function matchExamples(rule: RobotsRule): { matches: string[]; misses: string[] } {
+	let lines: ReturnType<typeof directives>;
+	try {
+		lines = directives(rule).filter((l) => l.directive === (rule.directive === "allow" ? "Allow" : "Disallow"));
+	} catch {
+		return { matches: [], misses: [] };
+	}
+	const exceptions = rule.directive === "disallow" ? directives(rule).filter((l) => l.directive === "Allow") : [];
+	const hit = (p: string) => lines.some((l) => matchRaw(l.value, p)) && !exceptions.some((l) => matchRaw(l.value, p));
+	const matches = [...new Set(lines.flatMap((l) => samplePaths(l.value)))].filter(hit).slice(0, 4);
+	const near = new Set<string>();
+	for (const e of exceptions) for (const p of samplePaths(e.value).slice(0, 1)) near.add(p);
+	for (const l of lines) {
+		const literal = l.value.replace(/\$$/, "").replace(/\*/g, "");
+		const trimmed = literal.replace(/\/$/, "");
+		if (trimmed) {
+			near.add(trimmed);
+			near.add(`${trimmed}-index/`);
+			near.add(`/other${literal.startsWith("/") ? literal : `/${literal}`}`);
+			near.add(`${literal}x`.replace(/^([^/])/, "/$1"));
+			near.add(`${trimmed.toUpperCase()}/`);
+		}
+		near.add(`${literal || "/"}?page=2`.replace(/^([^/])/, "/$1"));
+	}
+	near.add("/");
+	near.add("/blog/a-post/");
+	near.add("/file.pdf?download=1");
+	const misses = [...near].filter((p) => p.startsWith("/") && !p.startsWith("//") && !hit(p) && !matches.includes(p)).slice(0, 3);
+	return { matches, misses };
+}