@lownoise-studio/rendershield 0.1.4

package/docs/deploy-cloudflare.md

@@ -0,0 +1,204 @@
+# Deploying RenderShield with Cloudflare
+
+This document explains how to deploy RenderShield in front of an existing
+single-page application (SPA) using Cloudflare.
+
+The behavior is intentionally simple:
+
+- Humans receive the normal SPA
+- Crawlers receive prerendered static HTML
+
+RenderShield does not execute JavaScript for bots and does not alter crawler
+behavior. It only guarantees that complete HTML exists and is served when
+appropriate.
+
+---
+
+## Requirements
+
+You will need:
+
+- A Cloudflare account
+- A domain using Cloudflare nameservers
+- An existing SPA hosted somewhere (Lovable, Vercel, Netlify, etc.)
+- RenderShield build output (dist-prerender/)
+
+RenderShield does not host your application. It only controls routing for
+crawler requests.
+
+---
+
+## Architecture overview
+
+At a high level:
+
+- Cloudflare proxies all traffic for your domain
+- A Worker inspects incoming requests
+- Known crawlers are routed to prerendered HTML files
+- All other traffic passes through to your SPA unchanged
+
+If prerendered output is missing or incomplete, RenderShield fails the build.
+
+---
+
+## DNS configuration
+
+In Cloudflare → DNS, create a single record:
+
+- Type: CNAME
+- Name: @ (or www if applicable)
+- Target: your SPA origin (for example: example.lovable.app)
+- Proxy status: ON
+
+The proxy must be enabled.  
+If it is disabled, the Worker will never execute.
+
+### DNS sanity check
+
+Cloudflare may import old records when you add a domain.
+
+If you see A records pointing to IPs you do not recognize, remove them.
+A minimal setup is preferred: one proxied CNAME.
+
+---
+
+## SSL configuration
+
+In Cloudflare → SSL/TLS:
+
+- Set SSL mode to Full or Full (strict)
+
+Avoid Flexible.  
+Flexible SSL commonly causes redirect loops between Cloudflare and your origin.
+
+---
+
+## Worker deployment
+
+1) Go to Workers & Pages in the Cloudflare dashboard  
+2) Create a new Worker  
+3) Paste the contents of: dist-prerender/worker.js  
+4) Save and deploy  
+
+The Worker does not require access to your application source code.
+
+---
+
+## Worker routing (required)
+
+Attach the Worker to your domain by adding a route:
+
+- yourdomain.com/*
+
+If you use www, also add:
+
+- www.yourdomain.com/*
+
+Without a route, the Worker will never run.
+
+---
+
+## Hosting prerendered output
+
+The Worker must be able to fetch prerendered files.
+
+You need a static origin that serves paths like:
+
+- /content/example/index.html
+- /sitemap.xml
+- /robots.txt
+
+Common options include:
+
+- Cloudflare Pages
+- Any static file host
+- Your existing host (if it supports static files)
+
+Requirement:
+A request for a prerendered path must return the HTML file directly.
+
+---
+
+## Sitemap and robots
+
+Ensure these files are publicly accessible:
+
+- https://yourdomain.com/sitemap.xml
+- https://yourdomain.com/robots.txt
+
+Submit the sitemap URL in Google Search Console.
+
+---
+
+## Verification
+
+RenderShield includes a verify command to guide testing.
+
+### Worker execution test
+
+Run:
+
+curl -I -H "User-Agent: GPTBot" https://yourdomain.com/content/example
+
+If debug headers are enabled, you should see headers similar to:
+
+- X-Bot-Detected: true
+- X-Prerender: true
+- X-Final-Path: /content/example/index.html
+
+If these headers are missing:
+- The Worker route may not be attached
+- The Cloudflare proxy may be disabled
+- DNS may still point to a different origin
+
+---
+
+## Content comparison
+
+Human request (SPA response):
+
+curl -s https://yourdomain.com/content/example | grep -i "<title>"
+
+Crawler request (prerendered HTML):
+
+curl -s -H "User-Agent: Googlebot" https://yourdomain.com/content/example | grep -i "<title>"
+
+The crawler response should contain the prerendered, route-specific title.
+
+---
+
+## Common issues
+
+### Worker does not appear to run
+
+Symptoms:
+- Site loads normally in a browser
+- Crawlers still receive the SPA shell
+
+Check:
+- Worker route exists
+- DNS proxy is enabled
+
+### 421 errors from origin
+
+This typically indicates a Host header mismatch.
+
+Ensure the Worker constructs a fresh request to the origin rather than
+forwarding incoming request headers directly.
+
+### Redirect loops
+
+Symptoms:
+- Browser reports “too many redirects”
+
+Fix:
+- Set SSL mode to Full or Full (strict)
+
+---
+
+## Final note
+
+RenderShield does not manipulate rankings, indexing, or crawler behavior.
+
+It guarantees one thing only:
+that crawlers receive complete, static HTML instead of an empty shell.

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@lownoise-studio/rendershield",
+  "version": "0.1.4",
+  "description": "Boring bot-aware prerendering: real HTML for bots, SPA for humans.",
+  "type": "module",
+  "license": "MIT",
+  "bin": {
+    "rendershield": "dist/cli.js"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "dev": "node --enable-source-maps dist/cli.js",
+    "start": "node dist/cli.js",
+    "prepack": "npm run build"
+  },
+  "files": [
+    "dist/**",
+    "src/**",
+    "docs/**",
+    "content/**",
+    "DEPLOY.md",
+    "README.md",
+    "LICENSE"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Lownoise-Studio/rendershield.git"
+  },
+  "homepage": "https://github.com/Lownoise-Studio/rendershield#readme",
+  "bugs": {
+    "url": "https://github.com/Lownoise-Studio/rendershield/issues"
+  },
+  "dependencies": {
+    "fast-glob": "^3.3.2",
+    "fs-extra": "^11.2.0",
+    "gray-matter": "^4.0.3",
+    "markdown-it": "^14.1.0"
+  },
+  "devDependencies": {
+    "@types/fs-extra": "^11.0.4",
+    "typescript": "^5.5.4"
+  }
+}

package/src/cli.ts

@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+import { cmdInit } from "./commands/init.js";
+import { cmdBuild } from "./commands/build.js";
+import { cmdVerify } from "./commands/verify.js";
+
+function printHelp() {
+  console.log(`
+RenderShield (v0) — boring bot-aware prerendering.
+
+Usage:
+  rendershield init
+  rendershield build
+  rendershield verify
+
+Notes:
+  - Config file: rendershield.config.json
+  - Content: content/blog/*.md (frontmatter required)
+  - Output: dist-prerender/
+`);
+}
+
+async function main() {
+  const cmd = process.argv[2]?.trim();
+
+  if (!cmd || cmd === "-h" || cmd === "--help") {
+    printHelp();
+    process.exit(0);
+  }
+
+  try {
+    if (cmd === "init") {
+      await cmdInit();
+      return;
+    }
+    if (cmd === "build") {
+      await cmdBuild();
+      return;
+    }
+    if (cmd === "verify") {
+      await cmdVerify();
+      return;
+    }
+
+    console.error(`Unknown command: ${cmd}\n`);
+    printHelp();
+    process.exit(1);
+  } catch (err: any) {
+    const msg = err?.message ? String(err.message) : String(err);
+    console.error(`\nRenderShield error: ${msg}\n`);
+    process.exit(1);
+  }
+}
+
+main();

package/src/commands/build.ts

@@ -0,0 +1,72 @@
+import fs from "fs-extra";
+import path from "node:path";
+import { loadConfig } from "../core/loadConfig.js";
+import { loadAllMarkdownDocs } from "../core/loadMarkdown.js";
+import { renderPageHtml } from "../core/renderHtml.js";
+import { generateSitemapXml } from "../core/generateSitemap.js";
+import { generateRobotsTxt } from "../core/generateRobots.js";
+import { generateWorkerJs } from "../core/generateWorker.js";
+import { validatePrerenderHtml } from "../core/validateOutput.js";
+
+function routeToOutDir(outDirAbs: string, routePath: string): string {
+  // /blog/slug -> outDir/blog/slug/index.html
+  const clean = routePath.replace(/^\//, "");
+  return path.join(outDirAbs, clean);
+}
+
+export async function cmdBuild(cwd = process.cwd()) {
+  const cfg = await loadConfig(cwd);
+  const outDirAbs = path.join(cwd, cfg.output.outDir);
+
+  // Clean output (boring + deterministic)
+  await fs.remove(outDirAbs);
+  await fs.ensureDir(outDirAbs);
+
+  const docs = await loadAllMarkdownDocs(cfg, cwd);
+
+  if (docs.length === 0) {
+    throw new Error("No markdown documents found. Check content paths/patterns.");
+  }
+
+  // Generate pages (validate BEFORE writing)
+  for (const doc of docs) {
+    const pageDir = routeToOutDir(outDirAbs, doc.routePath);
+    await fs.ensureDir(pageDir);
+
+    const outFile = path.join(pageDir, "index.html");
+    const html = renderPageHtml(cfg, doc);
+
+    validatePrerenderHtml({
+      html,
+      outFile,
+      routePath: doc.routePath,
+    });
+
+    await fs.writeFile(outFile, html, "utf8");
+  }
+
+  // sitemap.xml
+  if (cfg.sitemap.enabled) {
+    const sitemapXml = generateSitemapXml(cfg, docs);
+    const sitemapPath = path.join(outDirAbs, cfg.sitemap.path.replace(/^\//, ""));
+    await fs.writeFile(sitemapPath, sitemapXml, "utf8");
+  }
+
+  // robots.txt
+  if (cfg.robots.enabled) {
+    const robotsTxt = generateRobotsTxt(cfg);
+    const robotsPath = path.join(outDirAbs, cfg.robots.path.replace(/^\//, ""));
+    await fs.writeFile(robotsPath, robotsTxt, "utf8");
+  }
+
+  // worker.js
+  if (cfg.worker.enabled) {
+    const workerJs = generateWorkerJs(cfg);
+    await fs.writeFile(path.join(outDirAbs, "worker.js"), workerJs, "utf8");
+  }
+
+  console.log(`Built ${docs.length} pages into ${cfg.output.outDir}/`);
+  console.log(
+    `Output includes: ${cfg.sitemap.enabled ? "sitemap.xml " : ""}${cfg.robots.enabled ? "robots.txt " : ""}${cfg.worker.enabled ? "worker.js" : ""}`
+  );
+}

package/src/commands/init.ts

@@ -0,0 +1,112 @@
+import fs from "fs-extra";
+import path from "node:path";
+
+const CONFIG_NAME = "rendershield.config.json";
+
+const DEFAULT_CONFIG = {
+  version: 1,
+  site: {
+    canonicalBase: "https://example.com",
+    siteName: "Example Site",
+    defaultOgImage: "https://example.com/og/default.jpg",
+    authorName: "Your Name"
+  },
+  content: {
+    markdown: {
+      baseDir: "content",
+      collections: [
+        {
+          name: "pages",
+          pattern: "pages/**/*.md",
+          routeBase: "/pages",
+          schemaType: "Article"
+        }
+      ]
+    }
+  },
+  output: {
+    outDir: "dist-prerender",
+    prettyHtml: true
+  },
+  sitemap: {
+    enabled: true,
+    path: "/sitemap.xml"
+  },
+  robots: {
+    enabled: true,
+    path: "/robots.txt"
+  },
+  worker: {
+    enabled: true,
+    lovableOrigin: "https://YOUR_SITE.lovable.app",
+    rewriteRouteBases: ["/pages/"],
+    botUserAgentPatterns: [
+      "googlebot",
+      "bingbot",
+      "gptbot",
+      "claudebot",
+      "perplexitybot",
+      "twitterbot",
+      "facebookexternalhit",
+      "linkedinbot",
+      "slackbot",
+      "discordbot",
+      "whatsapp",
+      "telegram"
+    ],
+    debugHeaders: true
+  }
+};
+
+const today = new Date().toISOString().slice(0, 10);
+
+const SAMPLE_POST = `---
+title: Hello World
+excerpt: This is a sample page generated by RenderShield.
+datePublished: ${today}
+coverImage: /images/hello.jpg
+slug: hello-world
+---
+
+This is **RenderShield**.
+
+- Crawlers should see real HTML.
+- Humans keep the SPA.
+
+That's the whole trick.
+`;
+
+export async function cmdInit(cwd = process.cwd()) {
+  const configPath = path.join(cwd, CONFIG_NAME);
+  const contentDir = path.join(cwd, "content", "pages");
+  const samplePath = path.join(contentDir, "hello-world.md");
+
+  const configExists = await fs.pathExists(configPath);
+  if (!configExists) {
+    await fs.writeFile(
+      configPath,
+      JSON.stringify(DEFAULT_CONFIG, null, 2) + "\n",
+      "utf8"
+    );
+    console.log(`Created ${CONFIG_NAME}`);
+  } else {
+    console.log(`${CONFIG_NAME} already exists (leaving it alone)`);
+  }
+
+  await fs.ensureDir(contentDir);
+
+  const sampleExists = await fs.pathExists(samplePath);
+  if (!sampleExists) {
+    await fs.writeFile(samplePath, SAMPLE_POST, "utf8");
+    console.log(`Created sample content: content/pages/hello-world.md`);
+  } else {
+    console.log(`Sample content already exists (leaving it alone)`);
+  }
+
+  console.log(`
+Next:
+  1) Edit rendershield.config.json
+  2) Add content under content/pages/
+  3) Run: rendershield build
+`);
+}

package/src/commands/verify.ts

@@ -0,0 +1,114 @@
+import fs from "fs-extra";
+import path from "node:path";
+import { loadConfig } from "../core/loadConfig.js";
+
+function joinUrl(base: string, routePath: string): string {
+  const b = base.endsWith("/") ? base.slice(0, -1) : base;
+  const p = routePath.startsWith("/") ? routePath : `/${routePath}`;
+  return b + p;
+}
+
+async function findFirstIndexHtml(outDirAbs: string): Promise<string | null> {
+  const stack: string[] = [outDirAbs];
+
+  while (stack.length > 0) {
+    const current = stack.pop() as string;
+
+    let entries: fs.Dirent[];
+    try {
+      entries = await fs.readdir(current, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+
+    entries.sort((a, b) => a.name.localeCompare(b.name));
+
+    for (const entry of entries) {
+      const full = path.join(current, entry.name);
+
+      if (entry.isDirectory()) {
+        stack.push(full);
+        continue;
+      }
+
+      if (entry.isFile() && entry.name.toLowerCase() === "index.html") {
+        // Ignore index.html at the output root; prefer a routed page like
+        // <section>/<slug>/index.html (or deeper).
+        const rel = path.relative(outDirAbs, full);
+        const parts = rel.split(path.sep).filter(Boolean);
+        if (parts.length >= 2) return full;
+      }
+    }
+  }
+
+  return null;
+}
+
+function indexHtmlPathToRoute(outDirAbs: string, indexPathAbs: string): string {
+  const rel = path.relative(outDirAbs, indexPathAbs);
+  // rel: <section>/<slug>/index.html
+  const noFile = rel.replace(/index\.html$/i, "");
+  const normalized = noFile.split(path.sep).join("/").replace(/\/+$/, "");
+  return "/" + normalized.replace(/^\/+/, "");
+}
+
+export async function cmdVerify(cwd = process.cwd()) {
+  const cfg = await loadConfig(cwd);
+
+  const outDirAbs = path.join(cwd, cfg.output.outDir);
+  const exists = await fs.pathExists(outDirAbs);
+
+  if (!exists) {
+    console.log(`
+RenderShield verify
+
+No prerender output directory found: ${cfg.output.outDir}/
+
+Run:
+  node dist/cli.js build
+`);
+    return;
+  }
+
+  const firstIndex = await findFirstIndexHtml(outDirAbs);
+
+  if (!firstIndex) {
+    console.log(`
+RenderShield verify
+
+No prerendered pages found inside: ${cfg.output.outDir}/
+
+Run:
+  node dist/cli.js build
+`);
+    return;
+  }
+
+  const routePath = indexHtmlPathToRoute(outDirAbs, firstIndex);
+  const url = joinUrl(cfg.site.canonicalBase, routePath);
+
+  console.log(`
+RenderShield verify
+
+Using:
+  canonicalBase: ${cfg.site.canonicalBase}
+  routePath:     ${routePath}
+  output file:   ${path.relative(cwd, firstIndex)}
+
+Smoke tests:
+
+1) Human (usually SPA shell):
+  curl -s ${url} | grep -i "<title>"
+
+2) Bot (should see prerendered, route-specific title):
+  curl -s -H "User-Agent: Googlebot" ${url} | grep -i "<title>"
+
+3) Debug headers (Worker must be routed + proxy ON):
+  curl -I -H "User-Agent: GPTBot" ${url}
+
+Expected headers (if debugHeaders enabled):
+  X-Bot-Detected: true
+  X-Prerender: true
+  X-Final-Path: ${routePath.replace(/\/?$/, "/index.html")}
+`);
+}

package/src/core/generateRobots.ts

@@ -0,0 +1,18 @@
+import { RenderShieldConfig } from "../types.js";
+
+export function generateRobotsTxt(cfg: RenderShieldConfig): string {
+  const sitemapUrl = cfg.sitemap.enabled
+    ? `${cfg.site.canonicalBase.replace(/\/$/, "")}${cfg.sitemap.path}`
+    : "";
+
+  const lines = [
+    "User-agent: *",
+    "Allow: /",
+  ];
+
+  if (sitemapUrl) {
+    lines.push(`Sitemap: ${sitemapUrl}`);
+  }
+
+  return lines.join("\n") + "\n";
+}

package/src/core/generateSitemap.ts

@@ -0,0 +1,38 @@
+import { MarkdownDoc, RenderShieldConfig } from "../types.js";
+
+function joinUrl(base: string, pathname: string): string {
+  const b = base.endsWith("/") ? base.slice(0, -1) : base;
+  const p = pathname.startsWith("/") ? pathname : `/${pathname}`;
+  return b + p;
+}
+
+function escapeXml(s: string): string {
+  return s
+    .replaceAll("&", "&")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;")
+    .replaceAll("'", "&apos;");
+}
+
+export function generateSitemapXml(cfg: RenderShieldConfig, docs: MarkdownDoc[]): string {
+  const urls = docs.map((d) => ({
+    loc: joinUrl(cfg.site.canonicalBase, d.routePath),
+    lastmod: d.datePublished,
+  }));
+
+  const items = urls
+    .map(
+      (u) => `  <url>
+    <loc>${escapeXml(u.loc)}</loc>
+    <lastmod>${escapeXml(u.lastmod)}</lastmod>
+  </url>`
+    )
+    .join("\n");
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${items}
+</urlset>
+`;
+}