@coywolf/emdash 0.35.13 → 0.35.15

package/src/redirects/slug-core.test.ts

@@ -1,50 +0,0 @@
-/**
- * Run: node --test src/redirects/slug-core.test.ts
- */
-import assert from "node:assert/strict";
-import { test } from "node:test";
-
-// @ts-expect-error -- Node's type stripping needs the real .ts extension.
-import { SLUG_NOTE_PREFIX, planSlugChange } from "./slug-core.ts";
-
-const ours = (id: string, source: string, target: string) => ({ id, source, target, note: `${SLUG_NOTE_PREFIX} Post` });
-const manual = (id: string, source: string, target: string) => ({ id, source, target, note: null });
-
-test("a new slug: one rule from the old URL to the new one", () => {
-	assert.deepEqual(planSlugChange("/news/seo/old/", "/news/seo/new/", []), { kind: "apply", deleteIds: [], retargetIds: [], upsert: {} });
-});
-
-test("unchanged or unknown URLs do nothing", () => {
-	assert.equal(planSlugChange("/a/", "/a", []).kind, "none");
-	assert.equal(planSlugChange(null, "/a/", []).kind, "none");
-	assert.equal(planSlugChange("/a/", null, []).kind, "none");
-});
-
-test("a second rename points the first rule at the new URL (no chain)", () => {
-	const plan = planSlugChange("/b/", "/c/", [ours("r1", "/a", "/b/")]);
-	assert.deepEqual(plan, { kind: "apply", deleteIds: [], retargetIds: ["r1"], upsert: {} });
-});
-
-test("renaming back deletes the rule that would shadow the live URL", () => {
-	const plan = planSlugChange("/b/", "/a/", [ours("r1", "/a", "/b/")]);
-	assert.equal(plan.kind, "apply");
-	if (plan.kind !== "apply") return;
-	assert.deepEqual(plan.deleteIds, ["r1"]);
-	assert.deepEqual(plan.retargetIds, [], "the deleted rule isn't retargeted too");
-	assert.deepEqual(plan.upsert, {});
-});
-
-test("our existing rule for the old URL is updated, not duplicated", () => {
-	const plan = planSlugChange("/b/", "/c/", [ours("r2", "/b", "/somewhere/")]);
-	assert.equal(plan.kind, "apply");
-	if (plan.kind === "apply") assert.deepEqual(plan.upsert, { id: "r2" });
-});
-
-test("a hand-made rule for the old URL is left alone, and hand-made rules are never edited", () => {
-	const plan = planSlugChange("/b/", "/c/", [manual("m1", "/b", "/elsewhere/"), manual("m2", "/x", "/b/"), manual("m3", "/c", "/b/")]);
-	assert.equal(plan.kind, "apply");
-	if (plan.kind !== "apply") return;
-	assert.equal(plan.keptManual?.id, "m1");
-	assert.deepEqual(plan.deleteIds, []);
-	assert.deepEqual(plan.retargetIds, []);
-});

package/src/robots/rep.test.ts

@@ -1,335 +0,0 @@
-/**
- * REP conformance tests, ported from Google's robots_test.cc
- * (https://github.com/google/robotstxt) via Coywolf SEO's PHP suite, plus
- * the RFC 9309 examples. Run: node --test src/robots/*.test.ts
- */
-import assert from "node:assert/strict";
-import { test } from "node:test";
-
-// @ts-expect-error -- Node's type stripping needs the real .ts extension.
-import * as REP from "./rep.ts";
-
-const rep = REP as typeof import("./rep.js");
-const al = (robots: string, agent: string, url: string) => rep.oneAgentAllowed(robots, agent, url);
-
-test("GoogleOnly_SystemTest", () => {
-	const r1 = "user-agent: FooBot\ndisallow: /\n";
-	assert.equal(al("", "FooBot", ""), true);
-	assert.equal(al(r1, "", ""), true);
-	assert.equal(al(r1, "FooBot", ""), false);
-	assert.equal(al("", "", ""), true);
-});
-
-test("ID_LineSyntax_Line (missing colon)", () => {
-	const u = "http://foo.bar/x/y";
-	assert.equal(al("user-agent: FooBot\ndisallow: /\n", "FooBot", u), false);
-	assert.equal(al("foo: FooBot\nbar: /\n", "FooBot", u), true);
-	assert.equal(al("user-agent FooBot\ndisallow /\n", "FooBot", u), false);
-});
-
-test("ID_LineSyntax_Groups (group merging)", () => {
-	const g =
-		"allow: /foo/bar/\n\nuser-agent: FooBot\ndisallow: /\nallow: /x/\nuser-agent: BarBot\ndisallow: /\nallow: /y/\n\n\nallow: /w/\nuser-agent: BazBot\n\nuser-agent: FooBot\nallow: /z/\ndisallow: /\n";
-	assert.equal(al(g, "FooBot", "http://foo.bar/x/b"), true);
-	assert.equal(al(g, "FooBot", "http://foo.bar/z/d"), true);
-	assert.equal(al(g, "FooBot", "http://foo.bar/y/c"), false);
-	assert.equal(al(g, "BarBot", "http://foo.bar/y/c"), true);
-	assert.equal(al(g, "BarBot", "http://foo.bar/w/a"), true);
-	assert.equal(al(g, "BarBot", "http://foo.bar/z/d"), false);
-	assert.equal(al(g, "BazBot", "http://foo.bar/z/d"), true);
-	assert.equal(al(g, "FooBot", "http://foo.bar/foo/bar/"), false);
-	assert.equal(al(g, "BarBot", "http://foo.bar/foo/bar/"), false);
-	assert.equal(al(g, "BazBot", "http://foo.bar/foo/bar/"), false);
-});
-
-test("ID_LineSyntax_Groups_OtherRules", () => {
-	const o1 = "User-agent: BarBot\nSitemap: https://foo.bar/sitemap\nUser-agent: *\nDisallow: /\n";
-	assert.equal(al(o1, "FooBot", "http://foo.bar/"), false);
-	assert.equal(al(o1, "BarBot", "http://foo.bar/"), false);
-	const o2 = "User-agent: FooBot\nInvalid-Unknown-Line: unknown\nUser-agent: *\nDisallow: /\n";
-	assert.equal(al(o2, "FooBot", "http://foo.bar/"), false);
-	assert.equal(al(o2, "BarBot", "http://foo.bar/"), false);
-});
-
-test("ID_REPLineNamesCaseInsensitive", () => {
-	for (const r of [
-		"USER-AGENT: FooBot\nALLOW: /x/\nDISALLOW: /\n",
-		"user-agent: FooBot\nallow: /x/\ndisallow: /\n",
-		"uSeR-aGeNt: FooBot\nAlLoW: /x/\ndIsAlLoW: /\n",
-	]) {
-		assert.equal(al(r, "FooBot", "http://foo.bar/x/y"), true);
-		assert.equal(al(r, "FooBot", "http://foo.bar/a/b"), false);
-	}
-});
-
-test("ID_VerifyValidUserAgentsToObey", () => {
-	assert.equal(rep.isValidUserAgentToObey("Foobot"), true);
-	assert.equal(rep.isValidUserAgentToObey("Foobot-Bar"), true);
-	assert.equal(rep.isValidUserAgentToObey("Foo_Bar"), true);
-	for (const bad of ["", "ツ", "Foobot*", " Foobot ", "Foobot/2.1", "Foobot Bar"]) {
-		assert.equal(rep.isValidUserAgentToObey(bad), false, bad);
-	}
-});
-
-test("ID_UserAgentValueCaseInsensitive", () => {
-	for (const name of ["FOO BAR", "foo bar", "FoO bAr"]) {
-		const r = `User-Agent: ${name}\nAllow: /x/\nDisallow: /\n`;
-		for (const agent of ["Foo", "foo"]) {
-			assert.equal(al(r, agent, "http://foo.bar/x/y"), true);
-			assert.equal(al(r, agent, "http://foo.bar/a/b"), false);
-		}
-	}
-});
-
-test("GoogleOnly_AcceptUserAgentUpToFirstSpace", () => {
-	const sp = "User-Agent: *\nDisallow: /\nUser-Agent: Foo Bar\nAllow: /x/\nDisallow: /\n";
-	assert.equal(al(sp, "Foo", "http://foo.bar/x/y"), true);
-	assert.equal(al(sp, "Foo Bar", "http://foo.bar/x/y"), false);
-});
-
-test("ID_GlobalGroups_Secondary", () => {
-	const global = "user-agent: *\nallow: /\nuser-agent: FooBot\ndisallow: /\n";
-	const only = "user-agent: FooBot\nallow: /\nuser-agent: BarBot\ndisallow: /\nuser-agent: BazBot\ndisallow: /\n";
-	const u = "http://foo.bar/x/y";
-	assert.equal(al("", "FooBot", u), true);
-	assert.equal(al(global, "FooBot", u), false);
-	assert.equal(al(global, "BarBot", u), true);
-	assert.equal(al(only, "QuxBot", u), true);
-});
-
-test("ID_AllowDisallow_Value_CaseSensitive", () => {
-	assert.equal(al("user-agent: FooBot\ndisallow: /x/\n", "FooBot", "http://foo.bar/x/y"), false);
-	assert.equal(al("user-agent: FooBot\ndisallow: /X/\n", "FooBot", "http://foo.bar/x/y"), true);
-});
-
-test("ID_LongestMatch", () => {
-	const ux = "http://foo.bar/x/page.html";
-	assert.equal(al("user-agent: FooBot\ndisallow: /x/page.html\nallow: /x/\n", "FooBot", ux), false);
-	assert.equal(al("user-agent: FooBot\nallow: /x/page.html\ndisallow: /x/\n", "FooBot", ux), true);
-	assert.equal(al("user-agent: FooBot\nallow: /x/page.html\ndisallow: /x/\n", "FooBot", "http://foo.bar/x/"), false);
-	assert.equal(al("user-agent: FooBot\ndisallow: \nallow: \n", "FooBot", ux), true);
-	assert.equal(al("user-agent: FooBot\ndisallow: /\nallow: /\n", "FooBot", ux), true);
-	assert.equal(al("user-agent: FooBot\ndisallow: /x\nallow: /x/\n", "FooBot", "http://foo.bar/x"), false);
-	assert.equal(al("user-agent: FooBot\ndisallow: /x\nallow: /x/\n", "FooBot", "http://foo.bar/x/"), true);
-	assert.equal(al("user-agent: FooBot\ndisallow: /x/page.html\nallow: /x/page.html\n", "FooBot", ux), true);
-	assert.equal(al("user-agent: FooBot\nallow: /page\ndisallow: /*.html\n", "FooBot", "http://foo.bar/page.html"), false);
-	assert.equal(al("user-agent: FooBot\nallow: /page\ndisallow: /*.html\n", "FooBot", "http://foo.bar/page"), true);
-	assert.equal(al("user-agent: FooBot\nallow: /x/page.\ndisallow: /*.html\n", "FooBot", ux), true);
-	assert.equal(al("user-agent: FooBot\nallow: /x/page.\ndisallow: /*.html\n", "FooBot", "http://foo.bar/x/y.html"), false);
-	const grp = "User-agent: *\nDisallow: /x/\nUser-agent: FooBot\nDisallow: /y/\n";
-	assert.equal(al(grp, "FooBot", "http://foo.bar/x/page"), true);
-	assert.equal(al(grp, "FooBot", "http://foo.bar/y/page"), false);
-});
-
-test("ID_Encoding", () => {
-	assert.equal(
-		al(
-			"User-agent: FooBot\nDisallow: /\nAllow: /foo/bar?qux=taz&baz=http://foo.bar?tar&par\n",
-			"FooBot",
-			"http://foo.bar/foo/bar?qux=taz&baz=http://foo.bar?tar&par",
-		),
-		true,
-	);
-	const utf = "User-agent: FooBot\nDisallow: /\nAllow: /foo/bar/ツ\n";
-	assert.equal(al(utf, "FooBot", "http://foo.bar/foo/bar/%E3%83%84"), true);
-	assert.equal(al(utf, "FooBot", "http://foo.bar/foo/bar/ツ"), false);
-	const enc = "User-agent: FooBot\nDisallow: /\nAllow: /foo/bar/%E3%83%84\n";
-	assert.equal(al(enc, "FooBot", "http://foo.bar/foo/bar/%E3%83%84"), true);
-	assert.equal(al(enc, "FooBot", "http://foo.bar/foo/bar/ツ"), false);
-	const baz = "User-agent: FooBot\nDisallow: /\nAllow: /foo/bar/%62%61%7A\n";
-	assert.equal(al(baz, "FooBot", "http://foo.bar/foo/bar/baz"), false);
-	assert.equal(al(baz, "FooBot", "http://foo.bar/foo/bar/%62%61%7A"), true);
-});
-
-test("ID_SpecialCharacters", () => {
-	const s1 = "User-agent: FooBot\nDisallow: /foo/bar/quz\nAllow: /foo/*/qux\n";
-	assert.equal(al(s1, "FooBot", "http://foo.bar/foo/bar/quz"), false);
-	assert.equal(al(s1, "FooBot", "http://foo.bar/foo/quz"), true);
-	assert.equal(al(s1, "FooBot", "http://foo.bar/foo//quz"), true);
-	assert.equal(al(s1, "FooBot", "http://foo.bar/foo/bax/quz"), true);
-	const s2 = "User-agent: FooBot\nDisallow: /foo/bar$\nAllow: /foo/bar/qux\n";
-	assert.equal(al(s2, "FooBot", "http://foo.bar/foo/bar"), false);
-	assert.equal(al(s2, "FooBot", "http://foo.bar/foo/bar/qux"), true);
-	assert.equal(al(s2, "FooBot", "http://foo.bar/foo/bar/"), true);
-	assert.equal(al(s2, "FooBot", "http://foo.bar/foo/bar/baz"), true);
-	const s3 = "User-agent: FooBot\n# Disallow: /\nDisallow: /foo/quz#qux\nAllow: /\n";
-	assert.equal(al(s3, "FooBot", "http://foo.bar/foo/bar"), true);
-	assert.equal(al(s3, "FooBot", "http://foo.bar/foo/quz"), false);
-});
-
-test("GoogleOnly_IndexHTMLisDirectory", () => {
-	const idx = "User-Agent: *\nAllow: /allowed-slash/index.html\nDisallow: /\n";
-	assert.equal(al(idx, "foobot", "http://foo.com/allowed-slash/"), true);
-	assert.equal(al(idx, "foobot", "http://foo.com/allowed-slash/index.htm"), false);
-	assert.equal(al(idx, "foobot", "http://foo.com/allowed-slash/index.html"), true);
-	assert.equal(al(idx, "foobot", "http://foo.com/anyother-url"), false);
-});
-
-test("GoogleOnly_LineTooLong", () => {
-	const maxLine = 2083 * 8;
-	const maxLen = maxLine - "/x/".length - "disallow: ".length + 1;
-	const longline = "/x/" + "a".repeat(Math.max(0, maxLen - 3));
-	const rt = `user-agent: FooBot\ndisallow: ${longline}/qux\n`;
-	assert.equal(al(rt, "FooBot", "http://foo.bar/fux"), true);
-	assert.equal(al(rt, "FooBot", `http://foo.bar${longline}/fux`), false);
-
-	const maxLen2 = maxLine - "/x/".length - "allow: ".length + 1;
-	let la = "/x/";
-	let lb = "/x/";
-	while (la.length < maxLen2) {
-		la += "a";
-		lb += "b";
-	}
-	const rt2 = `user-agent: FooBot\ndisallow: /\nallow: ${la}/qux\nallow: ${lb}/qux\n`;
-	assert.equal(al(rt2, "FooBot", "http://foo.bar/"), false);
-	assert.equal(al(rt2, "FooBot", `http://foo.bar${la}/qux`), true);
-	assert.equal(al(rt2, "FooBot", `http://foo.bar${lb}/fux`), true);
-});
-
-test("GoogleOnly_DocumentationChecks", () => {
-	const base = "http://foo.bar";
-	const check = (robots: string, cases: Record<string, boolean>) => {
-		for (const [p, want] of Object.entries(cases)) assert.equal(al(robots, "FooBot", base + p), want, `${robots.split("\n")[2]} ${p}`);
-	};
-	check("user-agent: FooBot\ndisallow: /\nallow: /fish\n", {
-		"/bar": false, "/fish": true, "/fish.html": true, "/fish/salmon.html": true, "/fishheads": true,
-		"/fishheads/yummy.html": true, "/fish.html?id=anything": true, "/Fish.asp": false, "/catfish": false, "/?id=fish": false,
-	});
-	check("user-agent: FooBot\ndisallow: /\nallow: /fish*\n", {
-		"/bar": false, "/fish": true, "/fish.html": true, "/fish/salmon.html": true, "/fishheads": true,
-		"/fishheads/yummy.html": true, "/fish.html?id=anything": true, "/Fish.bar": false, "/catfish": false, "/?id=fish": false,
-	});
-	check("user-agent: FooBot\ndisallow: /\nallow: /fish/\n", {
-		"/fish/": true, "/fish/salmon": true, "/fish/?salmon": true, "/fish/salmon.html": true,
-		"/fish/?id=anything": true, "/fish": false, "/fish.html": false, "/Fish/Salmon.html": false,
-	});
-	check("user-agent: FooBot\ndisallow: /\nallow: /*.php\n", {
-		"/bar": false, "/filename.php": true, "/folder/filename.php": true, "/folder/filename.php?parameters": true,
-		"//folder/any.php.file.html": true, "/filename.php/": true, "/index?f=filename.php/": true, "/php/": false,
-		"/index?php": false, "/windows.PHP": false,
-	});
-	check("user-agent: FooBot\ndisallow: /\nallow: /*.php$\n", {
-		"/bar": false, "/filename.php": true, "/folder/filename.php": true, "/filename.php?parameters": false,
-		"/filename.php/": false, "/filename.php5": false, "/php/": false, "/filename?php": false, "/aaaphpaaa": false,
-		"//windows.PHP": false,
-	});
-	check("user-agent: FooBot\ndisallow: /\nallow: /fish*.php\n", {
-		"/fish.php": true, "/fishheads/catfish.php?parameters": true, "/Fish.PHP": false,
-	});
-	const ex = "http://example.com";
-	assert.equal(al("user-agent: FooBot\nallow: /p\ndisallow: /\n", "FooBot", `${ex}/page`), true);
-	assert.equal(al("user-agent: FooBot\nallow: /folder\ndisallow: /folder\n", "FooBot", `${ex}/folder/page`), true);
-	assert.equal(al("user-agent: FooBot\nallow: /page\ndisallow: /*.htm\n", "FooBot", `${ex}/page.htm`), false);
-	assert.equal(al("user-agent: FooBot\nallow: /$\ndisallow: /\n", "FooBot", `${ex}/`), true);
-	assert.equal(al("user-agent: FooBot\nallow: /$\ndisallow: /\n", "FooBot", `${ex}/page.html`), false);
-});
-
-test("TestGetPathParamsQuery", () => {
-	const cases: [string, string][] = [
-		["", "/"], ["http://www.example.com", "/"], ["http://www.example.com/", "/"], ["http://www.example.com/a", "/a"],
-		["http://www.example.com/a/", "/a/"], ["http://www.example.com/a/b?c=http://d.e/", "/a/b?c=http://d.e/"],
-		["http://www.example.com/a/b?c=d&e=f#fragment", "/a/b?c=d&e=f"], ["example.com", "/"], ["example.com/", "/"],
-		["example.com/a", "/a"], ["example.com/a/", "/a/"], ["example.com/a/b?c=d&e=f#fragment", "/a/b?c=d&e=f"],
-		["a", "/"], ["a/", "/"], ["/a", "/a"], ["a/b", "/b"], ["example.com?a", "/?a"], ["example.com/a;b#c", "/a;b"],
-		["//a/b/c", "/b/c"],
-	];
-	for (const [input, want] of cases) assert.equal(rep.pathParamsQuery(input), want, input);
-});
-
-test("TestMaybeEscapePattern", () => {
-	assert.equal(rep.escapePattern("http://www.example.com"), "http://www.example.com");
-	assert.equal(rep.escapePattern("/a/b/c"), "/a/b/c");
-	assert.equal(rep.escapePattern(rep.toBytes("á")), "%C3%A1");
-	assert.equal(rep.escapePattern("%aa"), "%AA");
-});
-
-test("parser: line endings, BOM, sitemap", () => {
-	const counts = (body: string) => {
-		const { directives } = rep.parse(body);
-		const valid = directives.filter((d: { type: string }) => d.type !== "unknown").length;
-		return { valid, unknown: directives.length - valid, last: directives.at(-1)?.line ?? 0 };
-	};
-	for (const body of [
-		"User-Agent: foo\nAllow: /some/path\nUser-Agent: bar\n\n\nDisallow: /\n",
-		"User-Agent: foo\r\nAllow: /some/path\r\nUser-Agent: bar\r\n\r\n\r\nDisallow: /\r\n",
-		"User-Agent: foo\rAllow: /some/path\rUser-Agent: bar\r\r\rDisallow: /\r",
-		"User-Agent: foo\nAllow: /some/path\nUser-Agent: bar\n\n\nDisallow: /",
-		"User-Agent: foo\nAllow: /some/path\r\nUser-Agent: bar\n\r\n\nDisallow: /",
-	]) {
-		const c = counts(body);
-		assert.equal(c.valid, 4);
-		assert.equal(c.last, 6);
-	}
-	assert.deepEqual(counts("User-Agent: foo\nAllow: /AnyValue\n").unknown, 0);
-	assert.deepEqual(counts("User-Agent: foo\nAllow: /AnyValue\n").valid, 2);
-	// (Google's partial-BOM cases are about raw byte streams; robots.txt arrives here as decoded text.)
-	const mid = counts("User-Agent: foo\nAllow: /AnyValue\n");
-	assert.equal(mid.valid, 1);
-	assert.equal(mid.unknown, 1);
-	const sitemap = (body: string) => rep.parse(body).directives.find((d: { type: string }) => d.type === "sitemap")?.value;
-	assert.equal(sitemap("User-Agent: foo\nAllow: /some/path\nUser-Agent: bar\n\n\nSitemap: http://foo.bar/sitemap.xml\n"), "http://foo.bar/sitemap.xml");
-	assert.equal(sitemap("Sitemap: http://foo.bar/sitemap.xml\nUser-Agent: foo\nAllow: /some/path\nUser-Agent: bar\n\n\n"), "http://foo.bar/sitemap.xml");
-});
-
-test("RFC 9309 §2.2.1: groups naming the same agent are merged", () => {
-	const r = "User-agent: ExampleBot\nDisallow: /foo\n\nUser-agent: *\nDisallow: /\n\nUser-agent: ExampleBot\nDisallow: /bar\n";
-	assert.equal(al(r, "ExampleBot", "https://example.com/foo/x"), false);
-	assert.equal(al(r, "ExampleBot", "https://example.com/bar/x"), false);
-	assert.equal(al(r, "ExampleBot", "https://example.com/baz"), true);
-	assert.equal(al(r, "OtherBot", "https://example.com/baz"), false);
-	// One group with several user-agent lines.
-	const multi = "User-agent: a\nUser-agent: b\nDisallow: /x\n";
-	assert.equal(al(multi, "a", "/x"), false);
-	assert.equal(al(multi, "b", "/x"), false);
-	assert.equal(al(multi, "c", "/x"), true);
-});
-
-test("RFC 9309 §2.2.2 / §5.2 examples: longest match, Allow on ties, * and $", () => {
-	const r = "User-Agent: *\nDisallow: *.gif$\nDisallow: /example/\nAllow: /publications/\n\nUser-Agent: foobot\nDisallow:/\nAllow:/example/page.html\nAllow:/example/allowed.gif\n";
-	assert.equal(al(r, "foobot", "/example/page.html"), true);
-	assert.equal(al(r, "foobot", "/example/allowed.gif"), true);
-	assert.equal(al(r, "foobot", "/example/other.html"), false);
-	assert.equal(al(r, "barbot", "/x/y.gif"), false);
-	assert.equal(al(r, "barbot", "/x/y.gif?q"), true);
-	assert.equal(al(r, "barbot", "/example/z"), false);
-	assert.equal(al(r, "barbot", "/publications/a"), true);
-	// /robots.txt itself is always fetchable in practice; the matcher just reports rules.
-	assert.equal(al("User-agent: *\nAllow: /a\nDisallow: /a\n", "x", "/a"), true);
-});
-
-test("evaluate() reports the deciding line", () => {
-	const v = rep.evaluate("User-agent: *\nDisallow: /private/\nAllow: /private/ok\n", ["GPTBot"], "https://x.test/private/secret");
-	assert.equal(v.allowed, false);
-	assert.equal(v.matchedDirective, "disallow");
-	assert.equal(v.matchedValue, "/private/");
-	assert.equal(v.matchedLine, 2);
-	assert.equal(v.scope, "global");
-});
-
-test("matches() normalizes both sides (rule tester)", () => {
-	const cases: [string, string, boolean][] = [
-		["/fish", "/fish", true], ["/fish", "/fish.html", true], ["/fish", "/Fish.asp", false], ["/fish", "/catfish", false],
-		["/fish/", "/fish", false], ["/*.php", "/x.php?p", true], ["/*.php$", "/x.php?p", false], ["/fish*.php", "/Fish.PHP", false],
-	];
-	for (const [p, path, want] of cases) assert.equal(rep.matches(p, path), want, `${p} ${path}`);
-	assert.equal(rep.matches("/caf%c3%a9", "/café"), true);
-});
-
-test("tokens with digits and dots match their own groups (MJ12bot, archive.org_bot)", () => {
-	const r = "User-agent: MJ12bot\nDisallow: /\n\nUser-agent: archive.org_bot\nDisallow: /private/\n\nUser-agent: MJ\nAllow: /\n";
-	assert.equal(al(r, "MJ12bot", "/page"), false);
-	assert.equal(al(r, "archive.org_bot", "/private/x"), false);
-	assert.equal(al(r, "archive.org_bot", "/public"), true);
-	assert.equal(al(r, "MJ", "/page"), true);
-	assert.equal(rep.extractProductToken("MJ12bot/1.4"), "MJ12bot");
-});
-
-test("tester path encoding: a typed /café matches /caf%C3%A9 rules", () => {
-	const r = "User-agent: *\nDisallow: /café\n";
-	assert.equal(rep.evaluate(r, ["x"], "/café", { encodePath: true }).allowed, false);
-	assert.equal(rep.evaluate(r, ["x"], "https://example.com/caf%c3%a9", { encodePath: true }).allowed, false);
-	assert.equal(rep.evaluate(r, ["x"], "/cafe", { encodePath: true }).allowed, true);
-	assert.equal(rep.evaluate("User-agent: *\nDisallow: /a%20b\n", ["x"], "/a b", { encodePath: true }).allowed, false);
-	// Google's contract (no encoding) is unchanged.
-	assert.equal(rep.evaluate(r, ["x"], "/café").allowed, true);
-});

package/src/robots/rules.test.ts

@@ -1,251 +0,0 @@
-/**
- * Generation, rule rendering, the bundled directory and the Radar merge.
- * Run: node --test src/robots/*.test.ts
- */
-import assert from "node:assert/strict";
-import { readFileSync } from "node:fs";
-import { test } from "node:test";
-
-// @ts-expect-error -- Node's type stripping needs the real .ts extension.
-import * as BOTS from "./bots.ts";
-// @ts-expect-error -- Node's type stripping needs the real .ts extension.
-import * as REP from "./rep.ts";
-// @ts-expect-error -- Node's type stripping needs the real .ts extension.
-import * as RULES from "./rules.ts";
-
-const rules = RULES as typeof import("./rules.js");
-const rep = REP as typeof import("./rep.js");
-const bots = BOTS as typeof import("./bots.js");
-type RobotsRule = import("./rules.js").RobotsRule;
-type BotEntry = import("./bots.js").BotEntry;
-
-const site = { siteUrl: "https://example.com" };
-let nextId = 0;
-const rule = (r: Partial<RobotsRule>): RobotsRule => ({
-	id: `r${++nextId}`,
-	name: "Rule",
-	enabled: true,
-	agents: ["*"],
-	directive: "disallow",
-	kind: "entire_site",
-	...r,
-});
-const config = (r: RobotsRule[], extra: Partial<import("./rules.js").RobotsConfig> = {}) => ({ ...rules.DEFAULT_CONFIG, rules: r, ...extra });
-
-test("rule kinds render like Coywolf SEO", () => {
-	const d = (r: Partial<RobotsRule>) => rules.directives(rule(r)).map((l) => `${l.directive}: ${l.value}`);
-	assert.deepEqual(d({ kind: "entire_site" }), ["Disallow: /"]);
-	assert.deepEqual(d({ kind: "folder", path: "private" }), ["Disallow: /private/"]);
-	assert.deepEqual(d({ kind: "prefix", path: "/draft" }), ["Disallow: /draft"]);
-	assert.deepEqual(d({ kind: "single_page", path: "/a.html", strict: true }), ["Disallow: /a.html$"]);
-	assert.deepEqual(d({ kind: "exact_url", path: "/search" }), ["Disallow: /search$"]);
-	assert.deepEqual(d({ kind: "filetype", ext: ".pdf" }), ["Disallow: /*.pdf$"]);
-	assert.deepEqual(d({ kind: "filetype_in_folder", path: "/dl", ext: "zip" }), ["Disallow: /dl/*.zip$"]);
-	assert.deepEqual(d({ kind: "contains", path: "/preview=" }), ["Disallow: /*preview="]);
-	assert.deepEqual(d({ kind: "any_depth", path: "/print/" }), ["Disallow: /*/print/"]);
-	assert.deepEqual(d({ kind: "query_any" }), ["Disallow: /*?"]);
-	assert.deepEqual(d({ kind: "query_param", path: "?utm_source" }), ["Disallow: /*?utm_source=", "Disallow: /*&utm_source="]);
-	assert.deepEqual(d({ kind: "wildcard_prefix", path: "/pa_" }), ["Disallow: /pa_*"]);
-	assert.deepEqual(d({ kind: "allow_exception", path: "/members", allow: "/members/hi" }), ["Disallow: /members/", "Allow: /members/hi"]);
-	assert.deepEqual(d({ kind: "folder", path: "/x/", directive: "allow" }), ["Allow: /x/"]);
-	assert.deepEqual(d({ kind: "custom", path: "/a b#c" }), ["Disallow: /abc"]);
-});
-
-test("validation", () => {
-	assert.throws(() => rules.validateRule(rule({ name: " " })), /name/);
-	assert.throws(() => rules.validateRule(rule({ agents: ["Bad Bot"] })), /token/);
-	assert.throws(() => rules.validateRule(rule({ kind: "folder", path: "" })), /path/);
-	assert.throws(() => rules.validateRule(rule({ kind: "custom", path: "x" })), /start with/);
-	assert.throws(() => rules.validateRule(rule({ kind: "filetype", ext: "p.d" })), /extension/);
-	rules.validateRule(rule({ agents: ["GPTBot", "archive.org_bot", "*"] }));
-});
-
-test("no rules: EmDash's defaults (admin blocked, media and sitemap kept)", () => {
-	const txt = rules.generate(config([]), site);
-	assert.match(txt, /User-agent: \*\nAllow: \/_emdash\/api\/media\/\nDisallow: \/_emdash\//);
-	assert.match(txt, /Sitemap: https:\/\/example\.com\/sitemap\.xml\n$/);
-	assert.equal(rep.oneAgentAllowed(txt, "Googlebot", "/_emdash/admin"), false);
-	assert.equal(rep.oneAgentAllowed(txt, "Googlebot", "/_emdash/api/media/file/a.jpg"), true);
-	assert.equal(rep.oneAgentAllowed(txt, "Googlebot", "/a-post/"), true);
-});
-
-test("bots named in rules still get EmDash's admin block; fully blocked bots don't get the media Allow", () => {
-	const txt = rules.generate(
-		config([
-			rule({ name: "Block AI training crawlers", agents: ["GPTBot", "CCBot"] }),
-			rule({ name: "Allow search engines", agents: ["Googlebot"], directive: "allow" }),
-			rule({ name: "No drafts for Bing", agents: ["bingbot"], kind: "folder", path: "/drafts/" }),
-		]),
-		site,
-	);
-	for (const bot of ["GPTBot", "CCBot"]) {
-		assert.equal(rep.oneAgentAllowed(txt, bot, "/"), false, bot);
-		assert.equal(rep.oneAgentAllowed(txt, bot, "/_emdash/api/media/file/a.jpg"), false, `${bot} media`);
-	}
-	assert.equal(rep.oneAgentAllowed(txt, "Googlebot", "/post/"), true);
-	assert.equal(rep.oneAgentAllowed(txt, "Googlebot", "/_emdash/admin"), false);
-	assert.equal(rep.oneAgentAllowed(txt, "Googlebot", "/_emdash/api/media/file/a.jpg"), true);
-	assert.equal(rep.oneAgentAllowed(txt, "bingbot", "/drafts/x"), false);
-	assert.equal(rep.oneAgentAllowed(txt, "bingbot", "/_emdash/api/content"), false);
-	assert.equal(rep.oneAgentAllowed(txt, "SomeOtherBot", "/drafts/x"), true);
-	assert.equal(rep.oneAgentAllowed(txt, "SomeOtherBot", "/_emdash/"), false);
-	assert.match(txt, /# Block AI training crawlers; discovery files stay readable\nUser-agent: GPTBot\nUser-agent: CCBot\nAllow: \/\.well-known\/\nDisallow: \/\n/);
-	const gpt = txt.slice(txt.indexOf("User-agent: GPTBot"), txt.indexOf("\n\n", txt.indexOf("User-agent: GPTBot")));
-	assert.ok(!gpt.includes("/_emdash/"));
-	for (const bot of ["Googlebot", "bingbot"]) {
-		const g = txt.slice(txt.indexOf(`User-agent: ${bot}`));
-		assert.ok(g.slice(0, g.indexOf("\n\n")).includes("Disallow: /_emdash/"), bot);
-	}
-});
-
-test("blocking everyone keeps media closed too", () => {
-	const txt = rules.generate(config([rule({ name: "Closed", agents: ["*"] })]), site);
-	assert.equal(rep.oneAgentAllowed(txt, "AnyBot", "/_emdash/api/media/file/a.jpg"), false);
-	assert.equal(rep.oneAgentAllowed(txt, "AnyBot", "/"), false);
-	assert.ok(!txt.includes("Allow: /_emdash/api/media/"));
-});
-
-test("disabled rules, comments off, sitemaps and extra lines", () => {
-	const txt = rules.generate(
-		config([rule({ name: "Off", enabled: false, agents: ["GPTBot"] })], {
-			comments: false,
-			includeSitemap: false,
-			allowMedia: false,
-			sitemaps: ["/news-sitemap.xml", "https://cdn.example.com/s.xml", ""],
-			extra: "Content-Signal: search=yes, ai-train=no",
-		}),
-		site,
-	);
-	assert.ok(!txt.includes("GPTBot"));
-	assert.ok(!txt.includes("#"));
-	assert.ok(!txt.includes("/sitemap.xml\n"));
-	assert.ok(!txt.includes("Allow:"));
-	assert.match(txt, /Content-Signal: search=yes, ai-train=no\n\nSitemap: https:\/\/example\.com\/news-sitemap\.xml\nSitemap: https:\/\/cdn\.example\.com\/s\.xml\n$/);
-});
-
-test("presets only use verified tokens from the bundled directory", () => {
-	const data = JSON.parse(readFileSync(new URL("./data/bots.json", import.meta.url), "utf8")) as { bots: BotEntry[] };
-	const verified = new Set(data.bots.filter((b) => b.status === "verified").map((b) => b.token));
-	for (const preset of rules.PRESETS) for (const agent of preset.agents) assert.ok(verified.has(agent), `${preset.name}: ${agent}`);
-});
-
-test("bundled directory: valid tokens, sources on verified entries, key crawlers verified", () => {
-	const data = JSON.parse(readFileSync(new URL("./data/bots.json", import.meta.url), "utf8")) as { bots: BotEntry[] };
-	assert.ok(data.bots.length > 650);
-	const slugs = new Set<string>();
-	const tokens = new Set<string>();
-	for (const b of data.bots) {
-		assert.ok(!tokens.has(b.token.toLowerCase()), `duplicate token ${b.token}`);
-		tokens.add(b.token.toLowerCase());
-	}
-	for (const b of data.bots) {
-		assert.ok(!slugs.has(b.slug), `duplicate ${b.slug}`);
-		slugs.add(b.slug);
-		assert.match(b.token, /^[A-Za-z0-9._-]+$/, b.slug);
-		assert.match(b.token, /[A-Za-z]/, b.slug);
-		if (b.evidence === "operator-docs") assert.doesNotMatch(b.sourceUrl ?? "", /^https?:\/\/[^/]+\/?$/, `${b.slug}: homepage isn't documentation`);
-		if (b.status === "verified") assert.ok(b.verifiedAt, `${b.slug} verifiedAt`);
-		if (b.evidence === "operator-docs") assert.match(b.sourceUrl ?? "", /^https?:\/\//, `${b.slug} source`);
-	}
-	const byToken = (t: string) => data.bots.find((b) => b.token === t && b.status === "verified") ?? data.bots.find((b) => b.token === t);
-	for (const t of [
-		"GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "Claude-SearchBot", "Google-Extended", "Googlebot",
-		"bingbot", "Applebot", "Applebot-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Bytespider", "Amazonbot",
-		"meta-externalagent", "DuckAssistBot", "YandexBot", "Baiduspider",
-	]) {
-		assert.equal(byToken(t)?.status, "verified", t);
-	}
-	for (const t of ["anthropic-ai", "cohere-ai", "FacebookBot"]) assert.equal(byToken(t)?.status, "unverified", t);
-});
-
-test("deriveToken from Radar patterns", () => {
-	assert.equal(bots.deriveToken(["Barkrowler/"], "Barkrowler"), "Barkrowler");
-	assert.equal(bots.deriveToken(["meta-externaltest/1\\.1"], "Meta-ExternalTest"), "meta-externaltest");
-	assert.equal(bots.deriveToken(["Advailo Kuma"], "Advailo Kuma"), null);
-	assert.equal(bots.deriveToken([], "AccessSparkBot"), "AccessSparkBot");
-	assert.equal(bots.deriveToken(["http://yandex.com/bots"], "Yandex Bot"), null);
-});
-
-test("mergeRadar stores only differences and adds new bots unverified", () => {
-	const base: BotEntry[] = [
-		{ slug: "gptbot", name: "GPTBot", operator: "OpenAI", category: "AI_CRAWLER", description: "d", token: "GPTBot", status: "verified", evidence: "operator-docs", origin: "radar" },
-		{ slug: "gone", name: "Gone", operator: "X", category: "OTHER", description: "", token: "GoneBot", status: "verified", evidence: "user-agent", origin: "radar" },
-		{ slug: "ccbot", name: "CCBot", operator: "Common Crawl", category: "AI_CRAWLER", description: "", token: "CCBot", status: "verified", evidence: "operator-docs", origin: "curated" },
-	];
-	const radar = [
-		{ slug: "gptbot", name: "GPTBot", operator: "OpenAI", category: "AI_CRAWLER", description: "d", userAgentPatterns: ["GPTBot"] },
-		{ slug: "newbot", name: "New Bot", operator: "N", category: "AI_SEARCH", description: "new", userAgentPatterns: ["NewBot/"] },
-	];
-	const first = bots.mergeRadar(base, [], radar, "2026-10-03");
-	assert.equal(first.added, 1);
-	assert.equal(first.delisted, 1); // "gone" left Radar; the curated CCBot is not Radar's to delist.
-	assert.deepEqual(first.writes.map((w) => w.slug).sort(), ["gone", "newbot"]);
-	const merged = bots.mergeDirectory(base, first.writes);
-	const nb = merged.find((b) => b.slug === "newbot");
-	assert.equal(nb?.token, "NewBot");
-	assert.equal(nb?.status, "unverified");
-	assert.equal(merged.find((b) => b.slug === "gone")?.delisted, true);
-
-	// Same data next week: nothing to write.
-	const second = bots.mergeRadar(base, first.writes, radar, "2026-10-10");
-	assert.equal(second.writes.length, 0);
-
-	// Radar recategorizes GPTBot: one overlay write, token untouched.
-	const third = bots.mergeRadar(base, first.writes, [{ ...radar[0], category: "AI_SEARCH" }, radar[1]], "2026-10-17");
-	assert.equal(third.updated, 1);
-	const after = bots.mergeDirectory(base, [...first.writes, ...third.writes]);
-	const g = after.find((b) => b.slug === "gptbot");
-	assert.equal(g?.category, "AI_SEARCH");
-	assert.equal(g?.token, "GPTBot");
-	assert.equal(g?.status, "verified");
-});
-
-test("whole-site blocks in any spelling drop the media Allow; /$ doesn't", () => {
-	for (const v of ["/", "/*", "*", "/**", "/*$"]) assert.equal(rules.blocksWholeSite(v), true, v);
-	assert.equal(rules.blocksWholeSite("/$"), false);
-	assert.equal(rules.blocksWholeSite("/private/"), false);
-	for (const v of ["/*", "*"]) {
-		const txt = rules.generate(config([rule({ name: "Block", agents: ["GPTBot"], kind: "custom", path: v })]), site);
-		assert.equal(rep.oneAgentAllowed(txt, "GPTBot", "/_emdash/api/media/file/a.jpg"), false, v);
-		const g = txt.slice(txt.indexOf("User-agent: GPTBot"));
-		assert.ok(!g.slice(0, g.indexOf("\n\n")).includes("/_emdash/"), v);
-	}
-	const home = rules.generate(config([rule({ name: "Home", agents: ["GPTBot"], kind: "custom", path: "/$" })]), site);
-	assert.equal(rep.oneAgentAllowed(home, "GPTBot", "/_emdash/api/media/file/a.jpg"), true);
-	assert.equal(rep.oneAgentAllowed(home, "GPTBot", "/_emdash/admin"), false);
-});
-
-test("line breaks can't inject directives", () => {
-	const txt = rules.generate(
-		config(
-			[
-				rule({
-					name: "Evil\nUser-agent: *\nDisallow: /",
-					description: "x\r\nAllow: /",
-					agents: ["GPTBot\nDisallow: /", "Good"],
-					kind: "folder",
-					path: "/a\nAllow: /b",
-				}),
-			],
-			{ sitemaps: ["/s.xml\nUser-agent: *\nDisallow: /", "https://x.test/a.xml\r\nDisallow: /"] },
-		),
-		{ siteUrl: "https://example.com\nDisallow: /" },
-	);
-	for (const line of txt.split("\n")) {
-		assert.ok(!/^Disallow: \/$/.test(line), `injected: ${line}`);
-		assert.ok(!/^User-agent: \*$/.test(line) || txt.includes("# EmDash admin"), line);
-	}
-	assert.ok(!txt.includes("\r"));
-	assert.equal(rep.oneAgentAllowed(txt, "Bingbot", "/post"), true);
-	assert.match(txt, /Sitemap: https:\/\/example\.comDisallow:\/s\.xmlUser-agent:\*Disallow:\//);
-});
-
-test("Radar entries merged by token count as their primary entry", () => {
-	const base: BotEntry[] = [
-		{ slug: "a", name: "A", operator: "", category: "OTHER", description: "", token: "ABot", status: "verified", evidence: "user-agent", origin: "radar", mergedSlugs: ["a-2"] },
-	];
-	const r = bots.mergeRadar(base, [], [{ slug: "a-2", name: "A two", userAgentPatterns: ["ABot"] }], "2026-10-03");
-	assert.equal(r.writes.length, 0);
-	assert.equal(r.added, 0);
-	assert.equal(r.delisted, 0);
-});