@onruntime/next-sitemap 0.4.0 → 0.5.0

package/README.md

@@ -17,6 +17,7 @@ Dynamic sitemap generation for Next.js with automatic route discovery.
 - Multi-sitemap support with sitemap index (for sites with >50,000 URLs)
 - hreflang alternates for i18n
 - Fully static generation (SSG)
+- robots.txt generation with Next.js `MetadataRoute.Robots` compatibility
 
 ## Installation
 
@@ -354,6 +355,47 @@ export async function GET() {
 </urlset>
 ```
 
+## Robots.txt (Pages Router)
+
+For Pages Router, use `createRobotsApiHandler` to generate a `robots.txt` file. The configuration uses the same format as [Next.js `MetadataRoute.Robots`](https://nextjs.org/docs/app/api-reference/file-conventions/metadata/robots).
+
+```typescript
+// pages/api/robots.txt.ts
+import { createRobotsApiHandler } from "@onruntime/next-sitemap/pages";
+
+export default createRobotsApiHandler({
+  rules: {
+    userAgent: "*",
+    allow: "/",
+    disallow: ["/admin", "/private"],
+  },
+  sitemap: "https://example.com/sitemap.xml",
+});
+```
+
+Add a rewrite in `next.config.ts`:
+
+```typescript
+{
+  source: "/robots.txt",
+  destination: "/api/robots.txt",
+}
+```
+
+**Note:** For App Router, use the [native `robots.ts` file convention](https://nextjs.org/docs/app/api-reference/file-conventions/metadata/robots) instead.
+
+## Requirements
+
+This package requires a **Node.js server runtime**. It works with:
+- Vercel
+- Netlify (with Next.js runtime)
+- Railway, Render, Fly.io
+- AWS (EC2, ECS, Lambda)
+- Docker containers
+- Any Node.js server
+
+It does **not** work with static exports (`output: 'export'`) or static hosting platforms like Cloudflare Pages, Netlify (static mode), or GitHub Pages.
+
 ## Troubleshooting
 
 ### Dynamic routes not included in sitemap

package/dist/app/index.cjs

@@ -4,7 +4,8 @@
 function calculateDepthPriority(pathname) {
   if (pathname === "/") return 1;
   const depth = pathname.split("/").filter(Boolean).length;
-  return Math.max(0.1, 1 - depth * 0.2);
+  const priority = Math.max(0.1, 1 - depth * 0.2);
+  return Math.round(priority * 100) / 100;
 }
 function shouldExclude(pathname, exclude) {
   if (!exclude) return false;
@@ -35,11 +36,16 @@ function getChangeFreq(pathname, changeFreq) {
   }
   return changeFreq;
 }
+function normalizePath(pathname) {
+  if (pathname === "/") return pathname;
+  return pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
+}
 function buildUrl(baseUrl, pathname, locale, defaultLocale) {
+  const normalizedPath = normalizePath(pathname);
   if (!locale || locale === defaultLocale) {
-    return `${baseUrl}${pathname}`;
+    return `${baseUrl}${normalizedPath}`;
   }
-  return `${baseUrl}/${locale}${pathname}`;
+  return `${baseUrl}/${locale}${normalizedPath}`;
 }
 function generateSitemapXml(entries) {
   const hasAlternates = entries.some((e) => e.alternates?.languages);

package/dist/app/index.js

@@ -2,7 +2,8 @@
 function calculateDepthPriority(pathname) {
   if (pathname === "/") return 1;
   const depth = pathname.split("/").filter(Boolean).length;
-  return Math.max(0.1, 1 - depth * 0.2);
+  const priority = Math.max(0.1, 1 - depth * 0.2);
+  return Math.round(priority * 100) / 100;
 }
 function shouldExclude(pathname, exclude) {
   if (!exclude) return false;
@@ -33,11 +34,16 @@ function getChangeFreq(pathname, changeFreq) {
   }
   return changeFreq;
 }
+function normalizePath(pathname) {
+  if (pathname === "/") return pathname;
+  return pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
+}
 function buildUrl(baseUrl, pathname, locale, defaultLocale) {
+  const normalizedPath = normalizePath(pathname);
   if (!locale || locale === defaultLocale) {
-    return `${baseUrl}${pathname}`;
+    return `${baseUrl}${normalizedPath}`;
   }
-  return `${baseUrl}/${locale}${pathname}`;
+  return `${baseUrl}/${locale}${normalizedPath}`;
 }
 function generateSitemapXml(entries) {
   const hasAlternates = entries.some((e) => e.alternates?.languages);

package/dist/index.cjs

@@ -4,7 +4,8 @@
 function calculateDepthPriority(pathname) {
   if (pathname === "/") return 1;
   const depth = pathname.split("/").filter(Boolean).length;
-  return Math.max(0.1, 1 - depth * 0.2);
+  const priority = Math.max(0.1, 1 - depth * 0.2);
+  return Math.round(priority * 100) / 100;
 }
 function shouldExclude(pathname, exclude) {
   if (!exclude) return false;
@@ -35,11 +36,16 @@ function getChangeFreq(pathname, changeFreq) {
   }
   return changeFreq;
 }
+function normalizePath(pathname) {
+  if (pathname === "/") return pathname;
+  return pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
+}
 function buildUrl(baseUrl, pathname, locale, defaultLocale) {
+  const normalizedPath = normalizePath(pathname);
   if (!locale || locale === defaultLocale) {
-    return `${baseUrl}${pathname}`;
+    return `${baseUrl}${normalizedPath}`;
   }
-  return `${baseUrl}/${locale}${pathname}`;
+  return `${baseUrl}/${locale}${normalizedPath}`;
 }
 function generateSitemapXml(entries) {
   const hasAlternates = entries.some((e) => e.alternates?.languages);
@@ -100,4 +106,5 @@ exports.generateSitemapIndexXml = generateSitemapIndexXml;
 exports.generateSitemapXml = generateSitemapXml;
 exports.getChangeFreq = getChangeFreq;
 exports.getPriority = getPriority;
+exports.normalizePath = normalizePath;
 exports.shouldExclude = shouldExclude;

package/dist/index.d.cts

@@ -92,6 +92,10 @@ interface RouteInfo {
 type PageModule = {
     generateStaticParams?: () => Promise<Record<string, string>[]>;
 };
+/**
+ * Normalize pathname by removing trailing slash (except for root)
+ */
+declare function normalizePath(pathname: string): string;
 /**
  * Generate the full URL for a pathname
  */
@@ -108,4 +112,4 @@ declare function generateSitemapIndexXml(baseUrl: string, sitemapCount: number,
     additionalSitemaps?: string[];
 }): string;
 
-export { type ChangeFrequency, type PageModule, type RouteInfo, type SitemapConfig, type SitemapEntry, buildUrl, calculateDepthPriority, generateSitemapIndexXml, generateSitemapXml, getChangeFreq, getPriority, shouldExclude };
+export { type ChangeFrequency, type PageModule, type RouteInfo, type SitemapConfig, type SitemapEntry, buildUrl, calculateDepthPriority, generateSitemapIndexXml, generateSitemapXml, getChangeFreq, getPriority, normalizePath, shouldExclude };

package/dist/index.d.ts

@@ -92,6 +92,10 @@ interface RouteInfo {
 type PageModule = {
     generateStaticParams?: () => Promise<Record<string, string>[]>;
 };
+/**
+ * Normalize pathname by removing trailing slash (except for root)
+ */
+declare function normalizePath(pathname: string): string;
 /**
  * Generate the full URL for a pathname
  */
@@ -108,4 +112,4 @@ declare function generateSitemapIndexXml(baseUrl: string, sitemapCount: number,
     additionalSitemaps?: string[];
 }): string;
 
-export { type ChangeFrequency, type PageModule, type RouteInfo, type SitemapConfig, type SitemapEntry, buildUrl, calculateDepthPriority, generateSitemapIndexXml, generateSitemapXml, getChangeFreq, getPriority, shouldExclude };
+export { type ChangeFrequency, type PageModule, type RouteInfo, type SitemapConfig, type SitemapEntry, buildUrl, calculateDepthPriority, generateSitemapIndexXml, generateSitemapXml, getChangeFreq, getPriority, normalizePath, shouldExclude };

package/dist/index.js

@@ -2,7 +2,8 @@
 function calculateDepthPriority(pathname) {
   if (pathname === "/") return 1;
   const depth = pathname.split("/").filter(Boolean).length;
-  return Math.max(0.1, 1 - depth * 0.2);
+  const priority = Math.max(0.1, 1 - depth * 0.2);
+  return Math.round(priority * 100) / 100;
 }
 function shouldExclude(pathname, exclude) {
   if (!exclude) return false;
@@ -33,11 +34,16 @@ function getChangeFreq(pathname, changeFreq) {
   }
   return changeFreq;
 }
+function normalizePath(pathname) {
+  if (pathname === "/") return pathname;
+  return pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
+}
 function buildUrl(baseUrl, pathname, locale, defaultLocale) {
+  const normalizedPath = normalizePath(pathname);
   if (!locale || locale === defaultLocale) {
-    return `${baseUrl}${pathname}`;
+    return `${baseUrl}${normalizedPath}`;
   }
-  return `${baseUrl}/${locale}${pathname}`;
+  return `${baseUrl}/${locale}${normalizedPath}`;
 }
 function generateSitemapXml(entries) {
   const hasAlternates = entries.some((e) => e.alternates?.languages);
@@ -92,4 +98,4 @@ ${allEntries}
 </sitemapindex>`;
 }
 
-export { buildUrl, calculateDepthPriority, generateSitemapIndexXml, generateSitemapXml, getChangeFreq, getPriority, shouldExclude };
+export { buildUrl, calculateDepthPriority, generateSitemapIndexXml, generateSitemapXml, getChangeFreq, getPriority, normalizePath, shouldExclude };

package/dist/pages/index.cjs

@@ -4,7 +4,8 @@
 function calculateDepthPriority(pathname) {
   if (pathname === "/") return 1;
   const depth = pathname.split("/").filter(Boolean).length;
-  return Math.max(0.1, 1 - depth * 0.2);
+  const priority = Math.max(0.1, 1 - depth * 0.2);
+  return Math.round(priority * 100) / 100;
 }
 function shouldExclude(pathname, exclude) {
   if (!exclude) return false;
@@ -35,11 +36,16 @@ function getChangeFreq(pathname, changeFreq) {
   }
   return changeFreq;
 }
+function normalizePath(pathname) {
+  if (pathname === "/") return pathname;
+  return pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
+}
 function buildUrl(baseUrl, pathname, locale, defaultLocale) {
+  const normalizedPath = normalizePath(pathname);
   if (!locale || locale === defaultLocale) {
-    return `${baseUrl}${pathname}`;
+    return `${baseUrl}${normalizedPath}`;
   }
-  return `${baseUrl}/${locale}${pathname}`;
+  return `${baseUrl}/${locale}${normalizedPath}`;
 }
 function generateSitemapXml(entries) {
   const hasAlternates = entries.some((e) => e.alternates?.languages);
@@ -94,6 +100,44 @@ ${allEntries}
 </sitemapindex>`;
 }
 
+// src/pages/robots.ts
+function generateRobotsTxt(config) {
+  const lines = [];
+  const rules = Array.isArray(config.rules) ? config.rules : [config.rules];
+  for (const rule of rules) {
+    const userAgents = Array.isArray(rule.userAgent) ? rule.userAgent : [rule.userAgent];
+    for (const userAgent of userAgents) {
+      lines.push(`User-agent: ${userAgent}`);
+    }
+    if (rule.allow) {
+      const allows = Array.isArray(rule.allow) ? rule.allow : [rule.allow];
+      for (const allow of allows) {
+        lines.push(`Allow: ${allow}`);
+      }
+    }
+    if (rule.disallow) {
+      const disallows = Array.isArray(rule.disallow) ? rule.disallow : [rule.disallow];
+      for (const disallow of disallows) {
+        lines.push(`Disallow: ${disallow}`);
+      }
+    }
+    if (rule.crawlDelay !== void 0) {
+      lines.push(`Crawl-delay: ${rule.crawlDelay}`);
+    }
+    lines.push("");
+  }
+  if (config.sitemap) {
+    const sitemaps = Array.isArray(config.sitemap) ? config.sitemap : [config.sitemap];
+    for (const sitemap of sitemaps) {
+      lines.push(`Sitemap: ${sitemap}`);
+    }
+  }
+  if (config.host) {
+    lines.push(`Host: ${config.host}`);
+  }
+  return lines.join("\n").trim() + "\n";
+}
+
 // src/pages/index.ts
 function extractRoutes(pagesContext, localeSegment) {
   const routes = [];
@@ -261,7 +305,16 @@ async function getSitemapStaticPaths(options) {
     fallback: false
   };
 }
+function createRobotsApiHandler(config) {
+  return function handler(_req, res) {
+    const robotsTxt = generateRobotsTxt(config);
+    res.setHeader("Content-Type", "text/plain");
+    res.status(200).send(robotsTxt);
+  };
+}
 
+exports.createRobotsApiHandler = createRobotsApiHandler;
 exports.createSitemapApiHandler = createSitemapApiHandler;
 exports.createSitemapIndexApiHandler = createSitemapIndexApiHandler;
+exports.generateRobotsTxt = generateRobotsTxt;
 exports.getSitemapStaticPaths = getSitemapStaticPaths;

package/dist/pages/index.d.cts

@@ -1,4 +1,5 @@
-import { NextApiRequest, NextApiResponse } from 'next';
+import { MetadataRoute, NextApiRequest, NextApiResponse } from 'next';
+export { MetadataRoute } from 'next';
 
 type ChangeFrequency = "always" | "hourly" | "daily" | "weekly" | "monthly" | "yearly" | "never";
 interface SitemapConfig {
@@ -72,6 +73,12 @@ type PageModule = {
     generateStaticParams?: () => Promise<Record<string, string>[]>;
 };
 
+/**
+ * Generate robots.txt content from configuration
+ * Compatible with Next.js MetadataRoute.Robots
+ */
+declare function generateRobotsTxt(config: MetadataRoute.Robots): string;
+
 interface CreateSitemapApiHandlerOptions extends SitemapConfig {
     /**
      * The require.context result for page discovery
@@ -110,5 +117,23 @@ declare function getSitemapStaticPaths(options: CreateSitemapApiHandlerOptions):
     }[];
     fallback: boolean;
 }>;
+/**
+ * Create API handler for robots.txt
+ * Use in: pages/api/robots.txt.ts or pages/robots.txt.ts (with rewrites)
+ *
+ * @example
+ * // pages/api/robots.txt.ts
+ * import { createRobotsApiHandler } from "@onruntime/next-sitemap/pages";
+ *
+ * export default createRobotsApiHandler({
+ *   rules: {
+ *     userAgent: "*",
+ *     allow: "/",
+ *     disallow: ["/admin", "/private"],
+ *   },
+ *   sitemap: "https://example.com/sitemap.xml",
+ * });
+ */
+declare function createRobotsApiHandler(config: MetadataRoute.Robots): (_req: NextApiRequest, res: NextApiResponse) => void;
 
-export { type ChangeFrequency, type CreateSitemapApiHandlerOptions, type SitemapConfig, type SitemapEntry, createSitemapApiHandler, createSitemapIndexApiHandler, getSitemapStaticPaths };
+export { type ChangeFrequency, type CreateSitemapApiHandlerOptions, type SitemapConfig, type SitemapEntry, createRobotsApiHandler, createSitemapApiHandler, createSitemapIndexApiHandler, generateRobotsTxt, getSitemapStaticPaths };

package/dist/pages/index.d.ts

@@ -1,4 +1,5 @@
-import { NextApiRequest, NextApiResponse } from 'next';
+import { MetadataRoute, NextApiRequest, NextApiResponse } from 'next';
+export { MetadataRoute } from 'next';
 
 type ChangeFrequency = "always" | "hourly" | "daily" | "weekly" | "monthly" | "yearly" | "never";
 interface SitemapConfig {
@@ -72,6 +73,12 @@ type PageModule = {
     generateStaticParams?: () => Promise<Record<string, string>[]>;
 };
 
+/**
+ * Generate robots.txt content from configuration
+ * Compatible with Next.js MetadataRoute.Robots
+ */
+declare function generateRobotsTxt(config: MetadataRoute.Robots): string;
+
 interface CreateSitemapApiHandlerOptions extends SitemapConfig {
     /**
      * The require.context result for page discovery
@@ -110,5 +117,23 @@ declare function getSitemapStaticPaths(options: CreateSitemapApiHandlerOptions):
     }[];
     fallback: boolean;
 }>;
+/**
+ * Create API handler for robots.txt
+ * Use in: pages/api/robots.txt.ts or pages/robots.txt.ts (with rewrites)
+ *
+ * @example
+ * // pages/api/robots.txt.ts
+ * import { createRobotsApiHandler } from "@onruntime/next-sitemap/pages";
+ *
+ * export default createRobotsApiHandler({
+ *   rules: {
+ *     userAgent: "*",
+ *     allow: "/",
+ *     disallow: ["/admin", "/private"],
+ *   },
+ *   sitemap: "https://example.com/sitemap.xml",
+ * });
+ */
+declare function createRobotsApiHandler(config: MetadataRoute.Robots): (_req: NextApiRequest, res: NextApiResponse) => void;
 
-export { type ChangeFrequency, type CreateSitemapApiHandlerOptions, type SitemapConfig, type SitemapEntry, createSitemapApiHandler, createSitemapIndexApiHandler, getSitemapStaticPaths };
+export { type ChangeFrequency, type CreateSitemapApiHandlerOptions, type SitemapConfig, type SitemapEntry, createRobotsApiHandler, createSitemapApiHandler, createSitemapIndexApiHandler, generateRobotsTxt, getSitemapStaticPaths };

package/dist/pages/index.js

@@ -2,7 +2,8 @@
 function calculateDepthPriority(pathname) {
   if (pathname === "/") return 1;
   const depth = pathname.split("/").filter(Boolean).length;
-  return Math.max(0.1, 1 - depth * 0.2);
+  const priority = Math.max(0.1, 1 - depth * 0.2);
+  return Math.round(priority * 100) / 100;
 }
 function shouldExclude(pathname, exclude) {
   if (!exclude) return false;
@@ -33,11 +34,16 @@ function getChangeFreq(pathname, changeFreq) {
   }
   return changeFreq;
 }
+function normalizePath(pathname) {
+  if (pathname === "/") return pathname;
+  return pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
+}
 function buildUrl(baseUrl, pathname, locale, defaultLocale) {
+  const normalizedPath = normalizePath(pathname);
   if (!locale || locale === defaultLocale) {
-    return `${baseUrl}${pathname}`;
+    return `${baseUrl}${normalizedPath}`;
   }
-  return `${baseUrl}/${locale}${pathname}`;
+  return `${baseUrl}/${locale}${normalizedPath}`;
 }
 function generateSitemapXml(entries) {
   const hasAlternates = entries.some((e) => e.alternates?.languages);
@@ -92,6 +98,44 @@ ${allEntries}
 </sitemapindex>`;
 }
 
+// src/pages/robots.ts
+function generateRobotsTxt(config) {
+  const lines = [];
+  const rules = Array.isArray(config.rules) ? config.rules : [config.rules];
+  for (const rule of rules) {
+    const userAgents = Array.isArray(rule.userAgent) ? rule.userAgent : [rule.userAgent];
+    for (const userAgent of userAgents) {
+      lines.push(`User-agent: ${userAgent}`);
+    }
+    if (rule.allow) {
+      const allows = Array.isArray(rule.allow) ? rule.allow : [rule.allow];
+      for (const allow of allows) {
+        lines.push(`Allow: ${allow}`);
+      }
+    }
+    if (rule.disallow) {
+      const disallows = Array.isArray(rule.disallow) ? rule.disallow : [rule.disallow];
+      for (const disallow of disallows) {
+        lines.push(`Disallow: ${disallow}`);
+      }
+    }
+    if (rule.crawlDelay !== void 0) {
+      lines.push(`Crawl-delay: ${rule.crawlDelay}`);
+    }
+    lines.push("");
+  }
+  if (config.sitemap) {
+    const sitemaps = Array.isArray(config.sitemap) ? config.sitemap : [config.sitemap];
+    for (const sitemap of sitemaps) {
+      lines.push(`Sitemap: ${sitemap}`);
+    }
+  }
+  if (config.host) {
+    lines.push(`Host: ${config.host}`);
+  }
+  return lines.join("\n").trim() + "\n";
+}
+
 // src/pages/index.ts
 function extractRoutes(pagesContext, localeSegment) {
   const routes = [];
@@ -259,5 +303,12 @@ async function getSitemapStaticPaths(options) {
     fallback: false
   };
 }
+function createRobotsApiHandler(config) {
+  return function handler(_req, res) {
+    const robotsTxt = generateRobotsTxt(config);
+    res.setHeader("Content-Type", "text/plain");
+    res.status(200).send(robotsTxt);
+  };
+}
 
-export { createSitemapApiHandler, createSitemapIndexApiHandler, getSitemapStaticPaths };
+export { createRobotsApiHandler, createSitemapApiHandler, createSitemapIndexApiHandler, generateRobotsTxt, getSitemapStaticPaths };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@onruntime/next-sitemap",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Dynamic sitemap generation for Next.js with automatic route discovery",
   "author": "onRuntime Studio <contact@onruntime.com>",
   "repository": {
@@ -45,6 +45,7 @@
     "@types/node": "^22.0.0",
     "next": "^16.1.1",
     "tsup": "^8",
+    "tsx": "^4",
     "typescript": "^5"
   },
   "keywords": [
@@ -60,6 +61,7 @@
   "scripts": {
     "build": "tsup",
     "dev": "tsup --watch",
-    "type-check": "tsc --noEmit"
+    "type-check": "tsc --noEmit",
+    "test": "tsx --test tests/index.ts"
   }
 }