@morphika/andami 0.1.7 → 0.1.8

package/app/(site)/[slug]/page.tsx

@@ -10,8 +10,9 @@ import { assetUrl } from "../../../lib/assets";
 
 const cfg = getSiteConfig();
 
-// ISR: cache for 1 hour, revalidate in background on next request.
-export const revalidate = 3600;
+// ISR: cache for 24 hours. Content changes are rare; admin can trigger
+// on-demand revalidation via /api/admin/revalidate after edits.
+export const revalidate = 86400;
 
 interface PageProps {
   params: Promise<{ slug: string }>;

package/app/(site)/page.tsx

@@ -4,8 +4,9 @@ import type { Page } from "../../lib/sanity/types";
 import { PageRenderer } from "../../components/blocks";
 import { getSiteConfig } from "../../lib/config";
 
-// ISR: cache for 1 hour, revalidate in background on next request.
-export const revalidate = 3600;
+// ISR: cache for 24 hours. Content changes are rare; admin can trigger
+// on-demand revalidation via /api/admin/revalidate after edits.
+export const revalidate = 86400;
 
 async function getHomePage(): Promise<Page | null> {
   try {

package/app/(site)/work/[slug]/page.tsx

@@ -10,9 +10,9 @@ import { assetUrl } from "../../../../lib/assets";
 
 const cfg = getSiteConfig();
 
-// ISR: cache for 1 hour, revalidate in background on next request.
-// Replaces force-dynamic — reduces SSR invocations by ~95%.
-export const revalidate = 3600;
+// ISR: cache for 24 hours. Content changes are rare; admin can trigger
+// on-demand revalidation via /api/admin/revalidate after edits.
+export const revalidate = 86400;
 
 interface ProjectPageProps {
   params: Promise<{ slug: string }>;

package/app/robots.ts

@@ -3,13 +3,50 @@ import { getSiteConfig } from "../lib/config";
 
 const cfg = getSiteConfig();
 
+/**
+ * robots.txt — Controls crawler access and rate.
+ *
+ * Crawl-delay (seconds between requests) is honoured by Bing, Yandex, Baidu
+ * and most well-behaved bots. Googlebot ignores it but respects the rate
+ * configured in Search Console. The 10-second delay drastically reduces
+ * serverless CPU usage from bot traffic on Hobby-tier hosting.
+ *
+ * Aggressive AI scrapers (GPTBot, CCBot, etc.) are blocked entirely.
+ */
 export default function robots(): MetadataRoute.Robots {
   return {
     rules: [
+      // Block known AI scrapers / aggressive bots
+      {
+        userAgent: "GPTBot",
+        disallow: ["/"],
+      },
+      {
+        userAgent: "CCBot",
+        disallow: ["/"],
+      },
+      {
+        userAgent: "anthropic-ai",
+        disallow: ["/"],
+      },
+      {
+        userAgent: "ClaudeBot",
+        disallow: ["/"],
+      },
+      {
+        userAgent: "Bytespider",
+        disallow: ["/"],
+      },
+      {
+        userAgent: "PetalBot",
+        disallow: ["/"],
+      },
+      // Default: allow with crawl delay
       {
         userAgent: "*",
         allow: "/",
-        disallow: ["/admin/", "/studio/", "/api/admin/"],
+        disallow: ["/admin/", "/studio/", "/api/admin/", "/api/"],
+        crawlDelay: 10,
       },
     ],
     sitemap: `${cfg.domain}/sitemap.xml`,

package/lib/bot-guard.ts

@@ -0,0 +1,138 @@
+/**
+ * Bot detection & rate-limiting for Edge Middleware.
+ *
+ * Runs on Vercel's Edge Runtime (cheap) to block aggressive crawlers
+ * BEFORE they invoke expensive serverless functions (Fluid Active CPU).
+ *
+ * Strategy:
+ * 1. Block known AI scrapers / aggressive bots by User-Agent
+ * 2. Detect bot-like rapid crawling patterns (many unique paths from same IP)
+ * 3. Return 429 for rate-limited requests
+ */
+
+import { NextRequest, NextResponse } from "next/server";
+
+// ── Known aggressive bot User-Agents ────────────────────────────────────
+// These bots ignore robots.txt or crawl too aggressively for Hobby-tier hosting.
+const BLOCKED_BOT_PATTERNS = [
+  "GPTBot",
+  "CCBot",
+  "anthropic-ai",
+  "ClaudeBot",
+  "Bytespider",
+  "PetalBot",
+  "Sogou",
+  "AhrefsBot",
+  "SemrushBot",
+  "DotBot",
+  "MJ12bot",
+  "BLEXBot",
+  "DataForSeoBot",
+  "serpstatbot",
+  "Amazonbot",
+  "Barkrowler",
+  "YandexBot",
+  "MegaIndex",
+  "Applebot",   // Apple's crawler — not needed for most portfolio sites
+];
+
+// ── Simple in-memory rate limiter for Edge ─────────────────────────────
+// Edge functions are short-lived, so this map resets frequently.
+// It won't catch all abuse but will throttle burst patterns within
+// a single Edge instance lifetime (typically several minutes).
+interface RateEntry {
+  count: number;
+  windowStart: number;
+}
+
+const ipHits = new Map<string, RateEntry>();
+
+// Max public page requests per IP per window (generous for humans, tight for bots)
+const PUBLIC_PAGE_LIMIT = 30;
+const WINDOW_MS = 60_000; // 1 minute
+
+// Garbage-collect stale entries every 100 checks
+let gcCounter = 0;
+
+function isRateLimited(ip: string): boolean {
+  const now = Date.now();
+
+  // Periodic cleanup
+  if (++gcCounter >= 100) {
+    gcCounter = 0;
+    for (const [key, entry] of ipHits) {
+      if (now - entry.windowStart > WINDOW_MS * 2) {
+        ipHits.delete(key);
+      }
+    }
+  }
+
+  const entry = ipHits.get(ip);
+  if (!entry || now - entry.windowStart > WINDOW_MS) {
+    ipHits.set(ip, { count: 1, windowStart: now });
+    return false;
+  }
+
+  entry.count++;
+  return entry.count > PUBLIC_PAGE_LIMIT;
+}
+
+// ── Main guard function ────────────────────────────────────────────────
+
+/**
+ * Call this at the TOP of your middleware, before any other logic.
+ * Returns a Response if the request should be blocked, or null to continue.
+ *
+ * Only applies to public (non-admin) GET requests for pages.
+ */
+export function guardAgainstBots(request: NextRequest): NextResponse | null {
+  const { pathname } = request.nextUrl;
+
+  // Skip admin routes, API routes (except public ones), and static assets
+  if (
+    pathname.startsWith("/admin") ||
+    pathname.startsWith("/api/admin") ||
+    pathname.startsWith("/studio") ||
+    pathname.startsWith("/_next") ||
+    pathname.startsWith("/fonts")
+  ) {
+    return null;
+  }
+
+  // Only guard GET requests (mutations already have their own rate limiter)
+  if (request.method !== "GET") {
+    return null;
+  }
+
+  const ua = request.headers.get("user-agent") || "";
+
+  // 1. Block known aggressive bots
+  const isBlockedBot = BLOCKED_BOT_PATTERNS.some(
+    (pattern) => ua.includes(pattern)
+  );
+
+  if (isBlockedBot) {
+    return new NextResponse("Forbidden", {
+      status: 403,
+      headers: { "X-Robots-Tag": "noindex, nofollow" },
+    });
+  }
+
+  // 2. Rate-limit public page requests per IP
+  const ip =
+    request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ||
+    request.headers.get("x-real-ip") ||
+    "unknown";
+
+  if (isRateLimited(ip)) {
+    return new NextResponse("Too Many Requests", {
+      status: 429,
+      headers: {
+        "Retry-After": "60",
+        "X-Robots-Tag": "noindex, nofollow",
+      },
+    });
+  }
+
+  return null;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@morphika/andami",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "description": "Visual Page Builder — core library. A reusable website builder with visual editing, CMS integration, and asset management.",
   "type": "module",
   "license": "MIT",
@@ -91,6 +91,7 @@
     "./lib/csrf": "./lib/csrf.ts",
     "./lib/csrf-client": "./lib/csrf-client.ts",
     "./lib/security": "./lib/security.ts",
+    "./lib/bot-guard": "./lib/bot-guard.ts",
     "./lib/sanitize": "./lib/sanitize.ts",
     "./lib/logger": "./lib/logger.ts",
     "./lib/audit": "./lib/audit.ts",