@tideworthy/tidysites-platform 0.10.11 → 0.10.12

package/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.10.12
+
+- The staging site is never indexed: every response carries `X-Robots-Tag: noindex, nofollow`, robots.txt disallows everything and the sitemap answers 404. The publish crawl still gets the real robots.txt and sitemap for the live copy.
+
 ## 0.10.11
 
 - Editor bar: Request a change comes first and carries the blue; Edit follows it plain.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@tideworthy/tidysites-platform",
-	"version": "0.10.11",
+	"version": "0.10.12",
 	"description": "The Tidysites platform integration for EmDash sites: bootstrap, sessions, maintenance, health, and publish hooks under /_tidy/*.",
 	"license": "MIT",
 	"type": "module",

package/src/middleware.ts

@@ -1,20 +1,44 @@
-/** Astro middleware registered by tidyPlatform(): the framing policy from TIDY_FRAME_ANCESTORS. */
+/**
+ * Astro middleware registered by tidyPlatform(): the framing policy from
+ * TIDY_FRAME_ANCESTORS, and no indexing of the staging site. This worker is
+ * the staging site (the live site is a static copy served by its own small
+ * worker), so every answer here carries noindex, robots.txt disallows
+ * everything and the sitemap is gone. The publish crawl identifies itself
+ * and gets the real robots.txt and sitemap, which is what the live copy
+ * ships.
+ */
 import type { MiddlewareHandler } from "astro";
 import { workerEnv } from "./lib/env";
 import { applyFramePolicy, frameAncestors } from "./lib/frame";
 
-export const onRequest: MiddlewareHandler = async (_context, next) => {
+const PUBLISH_CRAWL = "tidysites-publish";
+const NOINDEX = "noindex, nofollow";
+
+export const onRequest: MiddlewareHandler = async (context, next) => {
+	const crawl = context.request.headers.get("user-agent") === PUBLISH_CRAWL;
+	if (!crawl) {
+		const path = context.url.pathname;
+		if (path === "/robots.txt") {
+			return new Response("User-agent: *\nDisallow: /\n", { headers: { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store", "x-robots-tag": NOINDEX } });
+		}
+		if (path === "/sitemap.xml" || path === "/sitemap-index.xml" || /^\/sitemap-\d+\.xml$/.test(path)) {
+			return new Response("Not found", { status: 404, headers: { "cache-control": "no-store", "x-robots-tag": NOINDEX } });
+		}
+	}
 	const response = await next();
 	const env = await workerEnv();
 	const ancestors = frameAncestors(env?.TIDY_FRAME_ANCESTORS);
-	if (ancestors.length === 0) return response;
+	const apply = (headers: Headers) => {
+		if (ancestors.length > 0) applyFramePolicy(headers, ancestors);
+		if (!crawl) headers.set("x-robots-tag", NOINDEX);
+	};
 	// Response headers can be immutable (static assets, redirects); rebuild when so.
 	try {
-		applyFramePolicy(response.headers, ancestors);
+		apply(response.headers);
 		return response;
 	} catch {
 		const copy = new Response(response.body, response);
-		applyFramePolicy(copy.headers, ancestors);
+		apply(copy.headers);
 		return copy;
 	}
 };

package/src/version.ts

@@ -1,3 +1,3 @@
 /** Kept in step with package.json by the release checklist in MAINTAINING.md. */
-export const VERSION = "0.10.11";
+export const VERSION = "0.10.12";
 export const PLUGIN_ID = "tidy-platform";