@thomasfosterau/effect-robots 0.0.0-stage → 0.1.0

package/dist/index.js

@@ -0,0 +1,702 @@
+import * as Arr from "effect/Array";
+import * as Data from "effect/Data";
+import * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import * as Option from "effect/Option";
+import * as Predicate from "effect/Predicate";
+import * as Schema from "effect/Schema";
+import * as SchemaTransformation from "effect/SchemaTransformation";
+import * as Str from "effect/String";
+import * as HttpClient from "effect/http/HttpClient";
+//#region src/index.ts
+/**
+* `@thomasfosterau/effect-robots` — the Robots Exclusion Protocol
+* (RFC 9309) for Effect.
+*
+* Three layers, in one entry:
+*
+* 1. **The library**, its API and behaviour held fixed by the verbatim
+*    suite in `tests/library.test.ts`: the `Rule` / `RobotGroup` /
+*    `RobotsTxt` data classes, the `RobotsTxtFromString` schema,
+*    `parse` / `encode`, and the matcher `isAllowed` / `getCrawlDelay` /
+*    `getSitemaps`.
+* 2. **The consumer side** a crawler needs: {@link evaluateRobots} (the
+*    decision with its diagnostics), {@link fetchRobots} (RFC 9309 §2.3.1's
+*    failure posture over an `HttpClient`), {@link RobotsCache} (the
+*    pluggable per-origin cache; a persistent adapter, such as Workers KV,
+*    lives in the consuming application) and
+*    {@link makeRobotsChecker} (the cached `checkRobots` decision).
+* 3. **The serving side**: {@link robotsBody}, the stage-aware `robots.txt`
+*    a site serves.
+*
+* Peer: `effect` only.
+*
+* @since 0.1.0
+*/
+/**
+* A single rule in a robots.txt file: a path pattern and whether it allows.
+*
+* @example
+* ```ts
+* import { Rule } from "@thomasfosterau/effect-robots"
+*
+* const rule = new Rule({ path: "/admin", allow: false })
+* rule.path // "/admin"
+* ```
+*
+* @since 0.1.0
+* @category models
+*/
+var Rule = class extends Data.Class {};
+/**
+* A group of rules for one or more user agents.
+*
+* @example
+* ```ts
+* import { RobotGroup, Rule } from "@thomasfosterau/effect-robots"
+*
+* const group = new RobotGroup({ userAgents: ["*"], rules: [new Rule({ path: "/admin", allow: false })] })
+* group.crawlDelay // undefined
+* ```
+*
+* @since 0.1.0
+* @category models
+*/
+var RobotGroup = class extends Data.Class {};
+/**
+* A parsed robots.txt file: its groups and its `Sitemap:` URLs.
+*
+* @example
+* ```ts
+* import { RobotsTxt } from "@thomasfosterau/effect-robots"
+*
+* const robots = new RobotsTxt({ groups: [], sitemaps: ["https://example.com/sitemap.xml"] })
+* robots.sitemaps.length // 1
+* ```
+*
+* @since 0.1.0
+* @category models
+*/
+var RobotsTxt = class extends Data.Class {};
+/**
+* Validates that a value is a {@link Rule} instance (`Schema.instanceOf`).
+*
+* @since 0.1.0
+* @category schemas
+*/
+const RuleSchema = Schema.instanceOf(Rule, { identifier: "Rule" });
+/**
+* Validates that a value is a {@link RobotGroup} instance (`Schema.instanceOf`).
+*
+* @since 0.1.0
+* @category schemas
+*/
+const RobotGroupSchema = Schema.instanceOf(RobotGroup, { identifier: "RobotGroup" });
+/**
+* Validates that a value is a {@link RobotsTxt} instance (`Schema.instanceOf`); use {@link RobotsTxtFromString} to parse text.
+*
+* @since 0.1.0
+* @category schemas
+*/
+const RobotsTxtSchema = Schema.instanceOf(RobotsTxt, { identifier: "RobotsTxt" });
+/**
+* Splits a robots.txt line into its directive field and value, dropping
+* comments; `None` for a blank, comment-only or malformed line.
+*/
+const parseDirectiveLine = (rawLine) => {
+	const commentIndex = rawLine.indexOf("#");
+	const line = (commentIndex >= 0 ? rawLine.slice(0, commentIndex) : rawLine).trim();
+	if (Str.isEmpty(line)) return Option.none();
+	const colon = line.indexOf(":");
+	if (colon < 0) return Option.none();
+	return Option.some({
+		field: line.slice(0, colon).trim().toLowerCase(),
+		value: line.slice(colon + 1).trim()
+	});
+};
+/** Closes the group being accumulated, if it has any directive. */
+const finishGroup = (state) => {
+	if (Arr.isReadonlyArrayNonEmpty(state.userAgents) && state.hasDirectives) state.groups.push(new RobotGroup({
+		userAgents: state.userAgents,
+		rules: state.rules,
+		crawlDelay: state.crawlDelay
+	}));
+	if (state.hasDirectives) {
+		state.userAgents = [];
+		state.rules = [];
+		state.crawlDelay = void 0;
+		state.hasDirectives = false;
+	}
+};
+/** Applies one directive to the parser state (RFC 9309 group rules). */
+const applyDirective = (state, { field, value }) => {
+	if (Str.isEmpty(value)) return;
+	if (field === "user-agent") {
+		if (state.hasDirectives) finishGroup(state);
+		state.userAgents.push(value);
+		return;
+	}
+	if (field === "sitemap") {
+		state.sitemaps.push(value);
+		return;
+	}
+	if (field === "disallow" || field === "allow") {
+		state.rules.push(new Rule({
+			path: value,
+			allow: field === "allow"
+		}));
+		state.hasDirectives = true;
+		return;
+	}
+	if (field === "crawl-delay") {
+		const delay = Number.parseFloat(value);
+		if (!Number.isNaN(delay) && delay >= 0) {
+			state.crawlDelay = delay;
+			state.hasDirectives = true;
+		}
+	}
+};
+/**
+* Parses robots.txt content per RFC 9309: consecutive `User-agent:` lines
+* share one group; rules belong to the most recent group; `#` starts a
+* comment; unknown directives are ignored; a group with no directive is
+* dropped.
+*/
+const parseRobotsTxt = (content) => {
+	const state = {
+		groups: [],
+		sitemaps: [],
+		userAgents: [],
+		rules: [],
+		crawlDelay: void 0,
+		hasDirectives: false
+	};
+	for (const rawLine of content.split(/\r?\n/)) {
+		const directive = parseDirectiveLine(rawLine);
+		if (Option.isSome(directive)) applyDirective(state, directive.value);
+	}
+	finishGroup(state);
+	return new RobotsTxt({
+		groups: state.groups,
+		sitemaps: state.sitemaps
+	});
+};
+/** Encodes a group as its `User-agent` / rule / `Crawl-delay` lines. */
+const encodeGroup = (group) => [
+	...Arr.map(group.userAgents, (userAgent) => `User-agent: ${userAgent}`),
+	...Arr.map(group.rules, (rule) => `${rule.allow ? "Allow" : "Disallow"}: ${rule.path}`),
+	...group.crawlDelay === void 0 ? [] : [`Crawl-delay: ${group.crawlDelay}`],
+	""
+];
+/** Encodes a `RobotsTxt` as text: each group, a blank line, then the sitemaps. */
+const encodeRobotsTxt = (robotsTxt) => [...Arr.flatMap(robotsTxt.groups, encodeGroup), ...Arr.map(robotsTxt.sitemaps, (sitemap) => `Sitemap: ${sitemap}`)].join("\n");
+/**
+* The schema between robots.txt text and {@link RobotsTxt}: decoding parses
+* (RFC 9309 grouping, comments, unknown directives ignored), encoding
+* serialises.
+*
+* @example
+* ```ts
+* import * as Schema from "effect/Schema"
+* import { RobotsTxtFromString } from "@thomasfosterau/effect-robots"
+*
+* const robots = Schema.decodeUnknownSync(RobotsTxtFromString)("User-agent: *\nDisallow: /admin")
+* robots.groups.length // 1
+* Schema.encodeSync(RobotsTxtFromString)(robots) // "User-agent: *\nDisallow: /admin\n"
+* ```
+*
+* @since 0.1.0
+* @category schemas
+*/
+const RobotsTxtFromString = Schema.String.pipe(Schema.decodeTo(RobotsTxtSchema, SchemaTransformation.transform({
+	decode: parseRobotsTxt,
+	encode: encodeRobotsTxt
+})));
+/**
+* Parses robots.txt content into a {@link RobotsTxt} (synchronous).
+*
+* @example
+* ```ts
+* import { parse } from "@thomasfosterau/effect-robots"
+*
+* const robots = parse("User-agent: *\nDisallow: /admin\nAllow: /admin/public\n\nSitemap: https://example.com/sitemap.xml")
+* robots.groups[0]?.rules.length // 2
+* robots.sitemaps // ["https://example.com/sitemap.xml"]
+* ```
+*
+* @since 0.1.0
+* @category parsing
+*/
+const parse = (content) => Schema.decodeUnknownSync(RobotsTxtFromString)(content);
+/**
+* Encodes a {@link RobotsTxt} back to robots.txt text (synchronous).
+*
+* @example
+* ```ts
+* import { encode, RobotGroup, RobotsTxt, Rule } from "@thomasfosterau/effect-robots"
+*
+* encode(new RobotsTxt({
+*   groups: [new RobotGroup({ userAgents: ["*"], rules: [new Rule({ path: "/admin", allow: false })] })],
+*   sitemaps: ["https://example.com/sitemap.xml"],
+* }))
+* // "User-agent: *\nDisallow: /admin\n\nSitemap: https://example.com/sitemap.xml"
+* ```
+*
+* @since 0.1.0
+* @category parsing
+*/
+const encode = (robotsTxt) => Schema.encodeSync(RobotsTxtFromString)(robotsTxt);
+/** Escapes regex metacharacters (not `*`, expanded separately). */
+const escapeRegex = (value) => value.replace(/[.+?^${}()|[\]\\]/g, "\\$&");
+/**
+* Whether a rule path matches a URL path: `*` matches any sequence, a
+* trailing `$` anchors the end (RFC 9309 §2.2.3). An empty path and `/`
+* match everything.
+*/
+const matchesPath = (rulePath, urlPath) => {
+	if (rulePath === "" || rulePath === "/") return true;
+	const anchored = rulePath.endsWith("$");
+	const body = (anchored ? rulePath.slice(0, -1) : rulePath).split("*").map(escapeRegex).join(".*");
+	return new RegExp(`^${body}${anchored ? "$" : ""}`).test(urlPath);
+};
+/**
+* The specificity of a group's claim on a user agent: the length of its
+* longest matching product token (case-insensitive, exact or prefix), `0`
+* for the `*` wildcard, `-1` for no match (RFC 9309 §2.2.1).
+*/
+const groupSpecificity = (group, normalizedUserAgent) => Arr.reduce(group.userAgents, -1, (specificity, userAgent) => {
+	const pattern = userAgent.toLowerCase();
+	if (pattern === "*") return Math.max(specificity, 0);
+	if (normalizedUserAgent.startsWith(pattern)) return Math.max(specificity, pattern.length);
+	return specificity;
+});
+/**
+* The group whose user-agent token most specifically matches — the
+* longest matching token wins, ties keep the first group.
+*/
+const findMatchingGroup = (robotsTxt, userAgent) => {
+	const normalized = userAgent.toLowerCase();
+	return Arr.reduce(robotsTxt.groups, {
+		group: Option.none(),
+		specificity: -1
+	}, (best, group) => {
+		const specificity = groupSpecificity(group, normalized);
+		return specificity > best.specificity ? {
+			group: Option.some(group),
+			specificity
+		} : best;
+	}).group;
+};
+/** True when `rule` takes precedence over `incumbent` for the same path. */
+const outranks = (rule, incumbent) => Option.isNone(incumbent) || rule.path.length > incumbent.value.path.length || rule.path.length === incumbent.value.path.length && rule.allow && !incumbent.value.allow;
+/** The winning rule for `path`: longest match wins, Allow wins ties (RFC 9309 §2.2.2). */
+const findMostSpecificRule = (rules, path) => Arr.reduce(rules, Option.none(), (best, rule) => matchesPath(rule.path, path) && outranks(rule, best) ? Option.some(rule) : best);
+/** The path without query or fragment, made absolute. */
+const normalizePath = (path) => {
+	const bare = path.split("?")[0]?.split("#")[0] ?? "";
+	return bare.startsWith("/") ? bare : `/${bare}`;
+};
+/**
+* Whether `userAgent` may access `urlPath` under the rules: the most
+* specific matching group, then its longest matching rule, Allow winning
+* ties; no matching group or rule means allowed. The query string and
+* fragment are stripped before matching (the library's documented
+* behaviour; {@link evaluateRobots} matches the query too, as RFC 9309
+* §2.2.2 does).
+*
+* @example
+* ```ts
+* import { isAllowed, parse } from "@thomasfosterau/effect-robots"
+*
+* const robots = parse("User-agent: *\nDisallow: /admin\nAllow: /admin/public")
+* isAllowed(robots, "MyBot", "/admin") // false
+* isAllowed(robots, "MyBot", "/admin/public") // true
+* isAllowed(robots, "MyBot", "/public") // true
+* ```
+*
+* @since 0.1.0
+* @category matching
+*/
+const isAllowed = (robotsTxt, userAgent, urlPath) => {
+	const group = findMatchingGroup(robotsTxt, userAgent);
+	if (Option.isNone(group)) return true;
+	const rule = findMostSpecificRule(group.value.rules, normalizePath(urlPath));
+	return Option.isNone(rule) ? true : rule.value.allow;
+};
+/**
+* The crawl delay (seconds) for `userAgent`, from its matching group.
+*
+* @example
+* ```ts
+* import { getCrawlDelay, parse } from "@thomasfosterau/effect-robots"
+*
+* const robots = parse("User-agent: Googlebot\nCrawl-delay: 5\nDisallow: /x\n\nUser-agent: *\nCrawl-delay: 10\nDisallow: /y")
+* getCrawlDelay(robots, "Googlebot") // 5
+* getCrawlDelay(robots, "OtherBot") // 10
+* ```
+*
+* @since 0.1.0
+* @category matching
+*/
+const getCrawlDelay = (robotsTxt, userAgent) => Option.getOrUndefined(Option.map(findMatchingGroup(robotsTxt, userAgent), (group) => group.crawlDelay));
+/**
+* Every `Sitemap:` URL in the file.
+*
+* @example
+* ```ts
+* import { getSitemaps, parse } from "@thomasfosterau/effect-robots"
+*
+* getSitemaps(parse("Sitemap: https://example.com/sitemap.xml")) // ["https://example.com/sitemap.xml"]
+* ```
+*
+* @since 0.1.0
+* @category matching
+*/
+const getSitemaps = (robotsTxt) => robotsTxt.sitemaps;
+/**
+* Upper bound applied to `Crawl-delay` so a hostile value cannot stall a
+* batch.
+*
+* @since 0.1.0
+* @category constants
+*/
+const MAX_CRAWL_DELAY_SECONDS = 30;
+/**
+* Evaluates a parsed robots.txt for a user-agent token and a path — the
+* crawlers' decision, with its diagnostics. Group selection: the longest
+* matching product token wins, the `*` group is the fallback, none means
+* allowed. Rule precedence: longest matching pattern wins, Allow wins ties.
+* Unlike {@link isAllowed}, the path is matched as given, query included
+* (RFC 9309 §2.2.2), and only a fragment is stripped. The crawl delay is
+* capped and a non-positive one is dropped.
+*
+* @example
+* ```ts
+* import * as Option from "effect/Option"
+* import { evaluateRobots, parse } from "@thomasfosterau/effect-robots"
+*
+* const robots = parse("User-agent: *\nCrawl-delay: 3600\nDisallow: /private/\nAllow: /private/public/")
+* evaluateRobots(robots, "mybot", "/private/secret")
+* // { allowed: false, matchedPattern: Option.some("/private/"), crawlDelaySeconds: Option.some(30) }
+* evaluateRobots(robots, "mybot", "/private/public/page").allowed // true
+* ```
+*
+* @since 0.1.0
+* @category evaluation
+*/
+const evaluateRobots = (robotsTxt, userAgentToken, path, options = {}) => {
+	const group = findMatchingGroup(robotsTxt, userAgentToken);
+	if (Option.isNone(group)) return {
+		allowed: true,
+		matchedPattern: Option.none(),
+		crawlDelaySeconds: Option.none()
+	};
+	const cap = options.maxCrawlDelaySeconds ?? 30;
+	const rule = findMostSpecificRule(group.value.rules, path.split("#")[0] ?? "");
+	return {
+		allowed: Option.isNone(rule) ? true : rule.value.allow,
+		matchedPattern: Option.map(rule, (matched) => matched.path),
+		crawlDelaySeconds: Option.filter(Option.fromUndefinedOr(group.value.crawlDelay), (delay) => delay > 0).pipe(Option.map((delay) => Math.min(delay, cap)))
+	};
+};
+/**
+* robots.txt responses are capped at this many bytes (RFC 9309 §2.5).
+*
+* @since 0.1.0
+* @category constants
+*/
+const MAX_ROBOTS_BYTES = 512e3;
+/**
+* Per-origin cache TTL for a fetched body or a 4xx (24 hours): a directive
+* change is honoured within a day.
+*
+* @since 0.1.0
+* @category constants
+*/
+const ROBOTS_CACHE_TTL_SECONDS = 86400;
+/**
+* Cache TTL for an unavailable robots.txt (one hour), so a misbehaving
+* origin is not probed on every capture.
+*
+* @since 0.1.0
+* @category constants
+*/
+const ROBOTS_UNAVAILABLE_TTL_SECONDS = 3600;
+/**
+* The cache value for a robots.txt that could not be read (5xx, transport
+* failure, timeout). The leading character is a deliberate literal NUL
+* byte — it can never collide with a real body. Written as the `\0` escape
+* rather than an embedded raw NUL byte: same runtime string, but a raw NUL
+* in the source makes tools that binary-sniff stop treating the file as
+* text. Changing the value orphans entries already in a persistent cache.
+*
+* @since 0.1.0
+* @category constants
+*/
+const ROBOTS_UNAVAILABLE_SENTINEL = "\0__robots_unavailable__";
+/**
+* The cache value for an origin with no restrictions (a 4xx, 404 included).
+*
+* @since 0.1.0
+* @category constants
+*/
+const ROBOTS_NO_RESTRICTIONS = "";
+/**
+* A cache that stores nothing: every check fetches. The default cache of
+* {@link makeRobotsChecker}.
+*
+* @example
+* ```ts
+* import * as Effect from "effect/Effect"
+* import * as Option from "effect/Option"
+* import { noRobotsCache } from "@thomasfosterau/effect-robots"
+*
+* await Effect.runPromise(noRobotsCache.put("https://example.com", "User-agent: *", 60))
+* Option.isNone(await Effect.runPromise(noRobotsCache.get("https://example.com"))) // true
+* ```
+*
+* @since 0.1.0
+* @category caches
+*/
+const noRobotsCache = {
+	get: () => Effect.succeedNone,
+	put: () => Effect.void
+};
+/**
+* An in-memory cache (a `Map`), for tests and single-process crawlers. TTLs
+* are recorded, not enforced.
+*
+* @example
+* ```ts
+* import * as Effect from "effect/Effect"
+* import { memoryRobotsCache } from "@thomasfosterau/effect-robots"
+*
+* const cache = memoryRobotsCache()
+* await Effect.runPromise(cache.put("https://example.com", "User-agent: *\nDisallow: /", 60))
+* cache.entries.get("https://example.com")?.ttlSeconds // 60
+* ```
+*
+* @since 0.1.0
+* @category caches
+*/
+const memoryRobotsCache = () => {
+	const entries = /* @__PURE__ */ new Map();
+	return {
+		entries,
+		get: (origin) => Effect.sync(() => Option.map(Option.fromNullishOr(entries.get(origin)), (entry) => entry.value)),
+		put: (origin, value, ttlSeconds) => Effect.sync(() => {
+			entries.set(origin, {
+				value,
+				ttlSeconds
+			});
+		})
+	};
+};
+/**
+* Fetches `<origin>/robots.txt` through the `HttpClient` in context and
+* folds the outcome into the cache value with RFC 9309 §2.3.1's posture: a
+* 2xx body (capped), {@link ROBOTS_NO_RESTRICTIONS} for a 4xx (404
+* included), and {@link ROBOTS_UNAVAILABLE_SENTINEL} for a 5xx, a transport
+* failure, a timeout or a body that cannot be read — the crawler must
+* assume it is not permitted until it can read the rules. Never fails.
+*
+* Redirect policy is the client's: provide one whose redirects are
+* revalidated against the SSRF block list (`@thomasfosterau/effect-web/net`).
+*
+* @example
+* ```ts
+* import * as Effect from "effect/Effect"
+* import * as HttpClient from "effect/http/HttpClient"
+* import * as HttpClientResponse from "effect/http/HttpClientResponse"
+* import { fetchRobots, ROBOTS_NO_RESTRICTIONS } from "@thomasfosterau/effect-robots"
+*
+* const notFound = HttpClient.make((request) =>
+*   Effect.succeed(HttpClientResponse.fromWeb(request, new Response("nope", { status: 404 }))),
+* )
+* await Effect.runPromise(
+*   fetchRobots("https://example.com", { userAgent: "mybot/1.0" }).pipe(
+*     Effect.provideService(HttpClient.HttpClient, notFound),
+*   ),
+* ) === ROBOTS_NO_RESTRICTIONS // true
+* ```
+*
+* @since 0.1.0
+* @category fetching
+*/
+const fetchRobots = (origin, options) => Effect.gen(function* () {
+	const response = yield* (yield* HttpClient.HttpClient).get(`${origin}/robots.txt`, { headers: {
+		"user-agent": options.userAgent,
+		accept: "text/plain,*/*;q=0.5"
+	} });
+	if (response.status >= 200 && response.status < 300) return (yield* response.text).slice(0, options.maxBytes ?? 512e3);
+	return response.status >= 500 ? ROBOTS_UNAVAILABLE_SENTINEL : "";
+}).pipe(Effect.timeout(options.timeout ?? Duration.seconds(10)), Effect.orElseSucceed(() => ROBOTS_UNAVAILABLE_SENTINEL));
+/**
+* The reason a disallowed decision carries for an unavailable robots.txt.
+*
+* @since 0.1.0
+* @category constants
+*/
+const ROBOTS_UNAVAILABLE_REASON = "robots.txt unavailable (server error) — treating as disallow";
+/**
+* Builds a crawler's `checkRobots`: whether it may fetch `url` under the
+* origin's robots.txt, reading the origin's body through the cache and
+* fetching on a miss ({@link fetchRobots}), then {@link evaluateRobots}
+* for the crawler's token. Never fails — robots problems surface as
+* `allowed: false` decisions, and cache problems are the adapter's to
+* degrade. The fetch outcome is cached with the TTL its posture warrants
+* (unavailability shorter), and a failed cache write is not a failed
+* decision.
+*
+* @example
+* ```ts
+* import * as Effect from "effect/Effect"
+* import * as HttpClient from "effect/http/HttpClient"
+* import * as HttpClientResponse from "effect/http/HttpClientResponse"
+* import { makeRobotsChecker, memoryRobotsCache } from "@thomasfosterau/effect-robots"
+*
+* const site = HttpClient.make((request) =>
+*   Effect.succeed(HttpClientResponse.fromWeb(request, new Response("User-agent: *\nDisallow: /secret/", { status: 200 }))),
+* )
+* const checkRobots = makeRobotsChecker({
+*   userAgent: "mybot/1.0 (+https://example.com)",
+*   userAgentToken: "mybot",
+*   cache: memoryRobotsCache(),
+* })
+* const decision = await Effect.runPromise(
+*   checkRobots("https://origin.example/secret/x").pipe(Effect.provideService(HttpClient.HttpClient, site)),
+* )
+* decision // { allowed: false, reason: "disallowed by robots.txt rule '/secret/'" }
+* ```
+*
+* @since 0.1.0
+* @category checking
+*/
+const makeRobotsChecker = (options) => {
+	const cache = options.cache ?? noRobotsCache;
+	const loadRobots = (origin) => Effect.gen(function* () {
+		const cached = yield* cache.get(origin);
+		if (Option.isSome(cached)) return cached.value;
+		const fetched = yield* fetchRobots(origin, options);
+		const ttl = fetched === "\0__robots_unavailable__" ? options.unavailableTtlSeconds ?? 3600 : options.cacheTtlSeconds ?? 86400;
+		yield* cache.put(origin, fetched, ttl);
+		return fetched;
+	});
+	return (url) => Effect.gen(function* () {
+		const parsed = new URL(url);
+		const body = yield* loadRobots(parsed.origin);
+		if (body === "\0__robots_unavailable__") return {
+			allowed: false,
+			reason: ROBOTS_UNAVAILABLE_REASON
+		};
+		if (body === "") return {
+			allowed: true,
+			crawlDelaySeconds: Option.none()
+		};
+		const path = options.matchQuery === false ? parsed.pathname : parsed.pathname + parsed.search;
+		const evaluation = evaluateRobots(parse(body), options.userAgentToken, path, options);
+		if (!evaluation.allowed) return {
+			allowed: false,
+			reason: `disallowed by robots.txt rule '${Option.getOrElse(evaluation.matchedPattern, () => "")}'`
+		};
+		return {
+			allowed: true,
+			crawlDelaySeconds: evaluation.crawlDelaySeconds
+		};
+	});
+};
+/**
+* The non-production body: a full disallow, no sitemap (ROBOTS-005), so a
+* `dev-<branch>` / `ci-<run>` / PR-preview deployment is never crawled.
+*
+* @since 0.1.0
+* @category serving
+*/
+const NON_PRODUCTION_BODY = [
+	"User-agent: *",
+	"Disallow: /",
+	""
+].join("\n");
+/**
+* The production body: open to all crawlers (`User-agent: *`, AI crawlers
+* included), the hygiene disallows, then the absolute sitemap URL
+* (ROBOTS-002 … -004).
+*
+* @example
+* ```ts
+* import { productionBody } from "@thomasfosterau/effect-robots"
+*
+* productionBody({ disallow: ["/login", "/*\/edit"], sitemapUrl: "https://example.com/sitemap.xml", comments: ["All crawlers welcome."] })
+* // "# All crawlers welcome.\nUser-agent: *\nDisallow: /login\nDisallow: /*\/edit\n\nSitemap: https://example.com/sitemap.xml\n"
+* ```
+*
+* @since 0.1.0
+* @category serving
+*/
+const productionBody = (policy) => [
+	...Arr.map(policy.comments ?? [], (comment) => `# ${comment}`),
+	encode(new RobotsTxt({
+		groups: [new RobotGroup({
+			userAgents: ["*"],
+			rules: Arr.map(policy.disallow, (path) => new Rule({
+				path,
+				allow: false
+			}))
+		})],
+		sitemaps: [policy.sitemapUrl]
+	})),
+	""
+].join("\n");
+/**
+* The `robots.txt` body for a stage: the production policy when
+* `production`, else {@link NON_PRODUCTION_BODY} (ROBOTS-005). Pure, so the
+* stage branch is unit-testable without a runtime; the caller decides the
+* stage (`PUBLIC_STAGE === "production"`, say) and builds the sitemap URL
+* from the canonical base, never the request origin.
+*
+* @example
+* ```ts
+* import { NON_PRODUCTION_BODY, robotsBody } from "@thomasfosterau/effect-robots"
+*
+* robotsBody(false, { disallow: ["/login"], sitemapUrl: "https://example.com/sitemap.xml" }) === NON_PRODUCTION_BODY // true
+* robotsBody(true, { disallow: ["/login"], sitemapUrl: "https://example.com/sitemap.xml" }).includes("Sitemap: https://example.com/sitemap.xml") // true
+* ```
+*
+* @since 0.1.0
+* @category serving
+*/
+const robotsBody = (production, policy) => production ? productionBody(policy) : NON_PRODUCTION_BODY;
+/**
+* The `Cache-Control` a low-churn generated document (`robots.txt`,
+* `sitemap.xml`, `feed.json`) is served with: the edge caches the whole
+* document, bounding any request-time enumeration behind it to at most one
+* per hour per edge (PERF-CACHE-002). A mutation that could change the
+* document purges its `Cache-Tag` rather than relying on the TTL alone.
+*
+* @since 0.1.0
+* @category serving
+*/
+const LOW_CHURN_CACHE_CONTROL = "public, max-age=3600, s-maxage=3600, stale-while-revalidate=86400";
+/**
+* A guard for the strings the checker treats as sentinels: a real body is
+* neither empty nor the unavailable marker.
+*
+* @example
+* ```ts
+* import {
+*   isRobotsBody,
+*   ROBOTS_NO_RESTRICTIONS,
+*   ROBOTS_UNAVAILABLE_SENTINEL,
+* } from "@thomasfosterau/effect-robots"
+*
+* isRobotsBody("User-agent: *\nDisallow: /admin") // true
+* isRobotsBody(ROBOTS_NO_RESTRICTIONS) // false
+* isRobotsBody(ROBOTS_UNAVAILABLE_SENTINEL) // false
+* ```
+*
+* @since 0.1.0
+* @category predicates
+*/
+const isRobotsBody = (value) => Predicate.isString(value) && value !== "" && value !== "\0__robots_unavailable__";
+//#endregion
+export { LOW_CHURN_CACHE_CONTROL, MAX_CRAWL_DELAY_SECONDS, MAX_ROBOTS_BYTES, NON_PRODUCTION_BODY, ROBOTS_CACHE_TTL_SECONDS, ROBOTS_NO_RESTRICTIONS, ROBOTS_UNAVAILABLE_REASON, ROBOTS_UNAVAILABLE_SENTINEL, ROBOTS_UNAVAILABLE_TTL_SECONDS, RobotGroup, RobotGroupSchema, RobotsTxt, RobotsTxtFromString, RobotsTxtSchema, Rule, RuleSchema, encode, evaluateRobots, fetchRobots, getCrawlDelay, getSitemaps, isAllowed, isRobotsBody, makeRobotsChecker, memoryRobotsCache, noRobotsCache, parse, productionBody, robotsBody };