dh-remixer-sdk 0.0.37 → 0.0.38-ce1e2d3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dh-remixer-sdk",
-  "version": "0.0.37",
+  "version": "0.0.38-ce1e2d3",
   "type": "module",
   "files": [
     "templates",

package/scripts/ssg-helmet/index.mjs

@@ -10,6 +10,7 @@ import {
   populateRoutesFromHtml,
 } from "./populate-sitemap.mjs";
 import { buildSitemapXml } from "./build-sitemap-xml.mjs";
+import { writeRobotsTxt } from "./robots-txt.mjs";
 import { writeHelmetFile } from "./fs-utils.mjs";
 import {
   ROOT_ROUTE,
@@ -82,16 +83,7 @@ async function main() {
   );
   console.log("[created] sitemap.xml");
 
-  await fs.writeFile(
-    path.join(REMIXER_BUILD_DIR, "robots.txt"),
-    `User-agent: *
-Allow: /
-
-Sitemap: https://${TARGET_DOMAIN}/sitemap.xml
-`,
-    "utf8",
-  );
-  console.log("[created] robots.txt");
+  await writeRobotsTxt(REMIXER_BUILD_DIR, TARGET_DOMAIN);
 
   await prerenderer.removeBootstrapBackup();
 

package/scripts/ssg-helmet/robots-txt.mjs

@@ -0,0 +1,39 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+// The robots.txt every forked template shipped before robots.txt became
+// AI-editable. Sites carrying it verbatim never had it customised, so it is
+// replaced with the indexable default rather than preserved.
+const LEGACY_TEMPLATE_DEFAULT = "User-agent: *\nDisallow: /";
+
+export function resolveRobotsTxt(existing, targetDomain) {
+  const sitemapLine = `Sitemap: https://${targetDomain}/sitemap.xml`;
+
+  if (typeof existing !== "string") {
+    return `User-agent: *\nAllow: /\n\n${sitemapLine}\n`;
+  }
+
+  const normalised = existing.replace(/\r\n/g, "\n");
+
+  if (normalised.trim() === LEGACY_TEMPLATE_DEFAULT) {
+    return `User-agent: *\nAllow: /\n\n${sitemapLine}\n`;
+  }
+
+  if (/^[ \t]*Sitemap:/im.test(normalised)) {
+    return normalised;
+  }
+
+  return `${normalised.replace(/\s*$/, "")}\n\n${sitemapLine}\n`;
+}
+
+export async function writeRobotsTxt(buildDir, targetDomain) {
+  const robotsPath = path.join(buildDir, "robots.txt");
+  const existing = await fs.readFile(robotsPath, "utf8").catch(() => null);
+
+  await fs.writeFile(
+    robotsPath,
+    resolveRobotsTxt(existing, targetDomain),
+    "utf8",
+  );
+  console.log("[created] robots.txt");
+}

package/scripts/ssg-helmet/robots-txt.test.mjs

@@ -0,0 +1,41 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { resolveRobotsTxt } from "./robots-txt.mjs";
+
+const DOMAIN = "domain.com";
+const SITEMAP = "Sitemap: https://domain.com/sitemap.xml";
+
+test("writes the indexable default when the build has no robots.txt", () => {
+  const out = resolveRobotsTxt(null, DOMAIN);
+  assert.equal(out, `User-agent: *\nAllow: /\n\n${SITEMAP}\n`);
+});
+
+test("replaces the untouched legacy template default", () => {
+  const out = resolveRobotsTxt("User-agent: *\nDisallow: /", DOMAIN);
+  assert.equal(out, `User-agent: *\nAllow: /\n\n${SITEMAP}\n`);
+});
+
+test("replaces the legacy default regardless of line endings or trailing space", () => {
+  const out = resolveRobotsTxt("User-agent: *\r\nDisallow: /\r\n\r\n", DOMAIN);
+  assert.equal(out, `User-agent: *\nAllow: /\n\n${SITEMAP}\n`);
+});
+
+test("preserves an AI-authored robots.txt and appends the sitemap", () => {
+  const authored = "User-agent: *\nDisallow: /admin/\nAllow: /";
+  const out = resolveRobotsTxt(authored, DOMAIN);
+  assert.equal(out, `${authored}\n\n${SITEMAP}\n`);
+});
+
+test("preserves an AI-authored Disallow: / that is not the legacy default", () => {
+  const authored = "User-agent: Googlebot\nDisallow: /";
+  const out = resolveRobotsTxt(authored, DOMAIN);
+  assert.match(out, /^User-agent: Googlebot\nDisallow: \/$/m);
+  assert.doesNotMatch(out, /Allow: \//);
+});
+
+test("does not duplicate an existing Sitemap directive", () => {
+  const authored = `User-agent: *\nAllow: /\n\nSitemap: https://other.example/sitemap.xml\n`;
+  const out = resolveRobotsTxt(authored, DOMAIN);
+  assert.equal(out, authored);
+  assert.equal(out.match(/Sitemap:/g).length, 1);
+});

package/templates/base/cleanup.json

@@ -3,7 +3,6 @@
   "index.tsx",
   "filemap.json",
   "public/.htaccess",
-  "public/robots.txt",
   ".gitignore",
   "lib/supabase.ts",
   "tsconfig.json",

package/templates/base/filemap.json

@@ -6,7 +6,6 @@
   "filemap.json": "filemap.json",
   "tsconfig.json": "tsconfig.json",
   "vite.config.ts": "vite.config.ts",
-  "robots.txt": "public/robots.txt",
   ".htaccess": "public/.htaccess",
   "context/LanguageContext.tsx": "context/LanguageContext.tsx",
   "remixer.ts": "lib/remixer.ts",

package/templates/base/robots.txt

@@ -1,2 +0,0 @@
-User-agent: *
-Disallow: /