@onruntime/next-sitemap 0.4.1 → 0.5.0

package/README.md

@@ -17,6 +17,7 @@ Dynamic sitemap generation for Next.js with automatic route discovery.
 - Multi-sitemap support with sitemap index (for sites with >50,000 URLs)
 - hreflang alternates for i18n
 - Fully static generation (SSG)
+- robots.txt generation with Next.js `MetadataRoute.Robots` compatibility
 
 ## Installation
 
@@ -354,6 +355,47 @@ export async function GET() {
 </urlset>
 ```
 
+## Robots.txt (Pages Router)
+
+For Pages Router, use `createRobotsApiHandler` to generate a `robots.txt` file. The configuration uses the same format as [Next.js `MetadataRoute.Robots`](https://nextjs.org/docs/app/api-reference/file-conventions/metadata/robots).
+
+```typescript
+// pages/api/robots.txt.ts
+import { createRobotsApiHandler } from "@onruntime/next-sitemap/pages";
+
+export default createRobotsApiHandler({
+  rules: {
+    userAgent: "*",
+    allow: "/",
+    disallow: ["/admin", "/private"],
+  },
+  sitemap: "https://example.com/sitemap.xml",
+});
+```
+
+Add a rewrite in `next.config.ts`:
+
+```typescript
+{
+  source: "/robots.txt",
+  destination: "/api/robots.txt",
+}
+```
+
+**Note:** For App Router, use the [native `robots.ts` file convention](https://nextjs.org/docs/app/api-reference/file-conventions/metadata/robots) instead.
+
+## Requirements
+
+This package requires a **Node.js server runtime**. It works with:
+- Vercel
+- Netlify (with Next.js runtime)
+- Railway, Render, Fly.io
+- AWS (EC2, ECS, Lambda)
+- Docker containers
+- Any Node.js server
+
+It does **not** work with static exports (`output: 'export'`) or static hosting platforms like Cloudflare Pages, Netlify (static mode), or GitHub Pages.
+
 ## Troubleshooting
 
 ### Dynamic routes not included in sitemap

package/dist/pages/index.cjs

@@ -100,6 +100,44 @@ ${allEntries}
 </sitemapindex>`;
 }
 
+// src/pages/robots.ts
+function generateRobotsTxt(config) {
+  const lines = [];
+  const rules = Array.isArray(config.rules) ? config.rules : [config.rules];
+  for (const rule of rules) {
+    const userAgents = Array.isArray(rule.userAgent) ? rule.userAgent : [rule.userAgent];
+    for (const userAgent of userAgents) {
+      lines.push(`User-agent: ${userAgent}`);
+    }
+    if (rule.allow) {
+      const allows = Array.isArray(rule.allow) ? rule.allow : [rule.allow];
+      for (const allow of allows) {
+        lines.push(`Allow: ${allow}`);
+      }
+    }
+    if (rule.disallow) {
+      const disallows = Array.isArray(rule.disallow) ? rule.disallow : [rule.disallow];
+      for (const disallow of disallows) {
+        lines.push(`Disallow: ${disallow}`);
+      }
+    }
+    if (rule.crawlDelay !== void 0) {
+      lines.push(`Crawl-delay: ${rule.crawlDelay}`);
+    }
+    lines.push("");
+  }
+  if (config.sitemap) {
+    const sitemaps = Array.isArray(config.sitemap) ? config.sitemap : [config.sitemap];
+    for (const sitemap of sitemaps) {
+      lines.push(`Sitemap: ${sitemap}`);
+    }
+  }
+  if (config.host) {
+    lines.push(`Host: ${config.host}`);
+  }
+  return lines.join("\n").trim() + "\n";
+}
+
 // src/pages/index.ts
 function extractRoutes(pagesContext, localeSegment) {
   const routes = [];
@@ -267,7 +305,16 @@ async function getSitemapStaticPaths(options) {
     fallback: false
   };
 }
+function createRobotsApiHandler(config) {
+  return function handler(_req, res) {
+    const robotsTxt = generateRobotsTxt(config);
+    res.setHeader("Content-Type", "text/plain");
+    res.status(200).send(robotsTxt);
+  };
+}
 
+exports.createRobotsApiHandler = createRobotsApiHandler;
 exports.createSitemapApiHandler = createSitemapApiHandler;
 exports.createSitemapIndexApiHandler = createSitemapIndexApiHandler;
+exports.generateRobotsTxt = generateRobotsTxt;
 exports.getSitemapStaticPaths = getSitemapStaticPaths;

package/dist/pages/index.d.cts

@@ -1,4 +1,5 @@
-import { NextApiRequest, NextApiResponse } from 'next';
+import { MetadataRoute, NextApiRequest, NextApiResponse } from 'next';
+export { MetadataRoute } from 'next';
 
 type ChangeFrequency = "always" | "hourly" | "daily" | "weekly" | "monthly" | "yearly" | "never";
 interface SitemapConfig {
@@ -72,6 +73,12 @@ type PageModule = {
     generateStaticParams?: () => Promise<Record<string, string>[]>;
 };
 
+/**
+ * Generate robots.txt content from configuration
+ * Compatible with Next.js MetadataRoute.Robots
+ */
+declare function generateRobotsTxt(config: MetadataRoute.Robots): string;
+
 interface CreateSitemapApiHandlerOptions extends SitemapConfig {
     /**
      * The require.context result for page discovery
@@ -110,5 +117,23 @@ declare function getSitemapStaticPaths(options: CreateSitemapApiHandlerOptions):
     }[];
     fallback: boolean;
 }>;
+/**
+ * Create API handler for robots.txt
+ * Use in: pages/api/robots.txt.ts or pages/robots.txt.ts (with rewrites)
+ *
+ * @example
+ * // pages/api/robots.txt.ts
+ * import { createRobotsApiHandler } from "@onruntime/next-sitemap/pages";
+ *
+ * export default createRobotsApiHandler({
+ *   rules: {
+ *     userAgent: "*",
+ *     allow: "/",
+ *     disallow: ["/admin", "/private"],
+ *   },
+ *   sitemap: "https://example.com/sitemap.xml",
+ * });
+ */
+declare function createRobotsApiHandler(config: MetadataRoute.Robots): (_req: NextApiRequest, res: NextApiResponse) => void;
 
-export { type ChangeFrequency, type CreateSitemapApiHandlerOptions, type SitemapConfig, type SitemapEntry, createSitemapApiHandler, createSitemapIndexApiHandler, getSitemapStaticPaths };
+export { type ChangeFrequency, type CreateSitemapApiHandlerOptions, type SitemapConfig, type SitemapEntry, createRobotsApiHandler, createSitemapApiHandler, createSitemapIndexApiHandler, generateRobotsTxt, getSitemapStaticPaths };

package/dist/pages/index.d.ts

@@ -1,4 +1,5 @@
-import { NextApiRequest, NextApiResponse } from 'next';
+import { MetadataRoute, NextApiRequest, NextApiResponse } from 'next';
+export { MetadataRoute } from 'next';
 
 type ChangeFrequency = "always" | "hourly" | "daily" | "weekly" | "monthly" | "yearly" | "never";
 interface SitemapConfig {
@@ -72,6 +73,12 @@ type PageModule = {
     generateStaticParams?: () => Promise<Record<string, string>[]>;
 };
 
+/**
+ * Generate robots.txt content from configuration
+ * Compatible with Next.js MetadataRoute.Robots
+ */
+declare function generateRobotsTxt(config: MetadataRoute.Robots): string;
+
 interface CreateSitemapApiHandlerOptions extends SitemapConfig {
     /**
      * The require.context result for page discovery
@@ -110,5 +117,23 @@ declare function getSitemapStaticPaths(options: CreateSitemapApiHandlerOptions):
     }[];
     fallback: boolean;
 }>;
+/**
+ * Create API handler for robots.txt
+ * Use in: pages/api/robots.txt.ts or pages/robots.txt.ts (with rewrites)
+ *
+ * @example
+ * // pages/api/robots.txt.ts
+ * import { createRobotsApiHandler } from "@onruntime/next-sitemap/pages";
+ *
+ * export default createRobotsApiHandler({
+ *   rules: {
+ *     userAgent: "*",
+ *     allow: "/",
+ *     disallow: ["/admin", "/private"],
+ *   },
+ *   sitemap: "https://example.com/sitemap.xml",
+ * });
+ */
+declare function createRobotsApiHandler(config: MetadataRoute.Robots): (_req: NextApiRequest, res: NextApiResponse) => void;
 
-export { type ChangeFrequency, type CreateSitemapApiHandlerOptions, type SitemapConfig, type SitemapEntry, createSitemapApiHandler, createSitemapIndexApiHandler, getSitemapStaticPaths };
+export { type ChangeFrequency, type CreateSitemapApiHandlerOptions, type SitemapConfig, type SitemapEntry, createRobotsApiHandler, createSitemapApiHandler, createSitemapIndexApiHandler, generateRobotsTxt, getSitemapStaticPaths };

package/dist/pages/index.js

@@ -98,6 +98,44 @@ ${allEntries}
 </sitemapindex>`;
 }
 
+// src/pages/robots.ts
+function generateRobotsTxt(config) {
+  const lines = [];
+  const rules = Array.isArray(config.rules) ? config.rules : [config.rules];
+  for (const rule of rules) {
+    const userAgents = Array.isArray(rule.userAgent) ? rule.userAgent : [rule.userAgent];
+    for (const userAgent of userAgents) {
+      lines.push(`User-agent: ${userAgent}`);
+    }
+    if (rule.allow) {
+      const allows = Array.isArray(rule.allow) ? rule.allow : [rule.allow];
+      for (const allow of allows) {
+        lines.push(`Allow: ${allow}`);
+      }
+    }
+    if (rule.disallow) {
+      const disallows = Array.isArray(rule.disallow) ? rule.disallow : [rule.disallow];
+      for (const disallow of disallows) {
+        lines.push(`Disallow: ${disallow}`);
+      }
+    }
+    if (rule.crawlDelay !== void 0) {
+      lines.push(`Crawl-delay: ${rule.crawlDelay}`);
+    }
+    lines.push("");
+  }
+  if (config.sitemap) {
+    const sitemaps = Array.isArray(config.sitemap) ? config.sitemap : [config.sitemap];
+    for (const sitemap of sitemaps) {
+      lines.push(`Sitemap: ${sitemap}`);
+    }
+  }
+  if (config.host) {
+    lines.push(`Host: ${config.host}`);
+  }
+  return lines.join("\n").trim() + "\n";
+}
+
 // src/pages/index.ts
 function extractRoutes(pagesContext, localeSegment) {
   const routes = [];
@@ -265,5 +303,12 @@ async function getSitemapStaticPaths(options) {
     fallback: false
   };
 }
+function createRobotsApiHandler(config) {
+  return function handler(_req, res) {
+    const robotsTxt = generateRobotsTxt(config);
+    res.setHeader("Content-Type", "text/plain");
+    res.status(200).send(robotsTxt);
+  };
+}
 
-export { createSitemapApiHandler, createSitemapIndexApiHandler, getSitemapStaticPaths };
+export { createRobotsApiHandler, createSitemapApiHandler, createSitemapIndexApiHandler, generateRobotsTxt, getSitemapStaticPaths };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@onruntime/next-sitemap",
-  "version": "0.4.1",
+  "version": "0.5.0",
   "description": "Dynamic sitemap generation for Next.js with automatic route discovery",
   "author": "onRuntime Studio <contact@onruntime.com>",
   "repository": {