aeo.js 0.0.1 → 0.0.2

package/dist/next.mjs

@@ -0,0 +1,1106 @@
+import { existsSync, mkdirSync, writeFileSync, readdirSync, statSync, readFileSync, copyFileSync } from 'fs';
+import { join, dirname, extname, relative } from 'path';
+import 'minimatch';
+import { createHash } from 'crypto';
+
+// src/core/robots.ts
+var AI_CRAWLERS = [
+  "GPTBot",
+  "OAI-SearchBot",
+  "ChatGPT-User",
+  "ClaudeBot",
+  "Claude-Web",
+  "anthropic-ai",
+  "PerplexityBot",
+  "Google-Extended",
+  "Gemini-Deep-Research",
+  "Bingbot",
+  "FacebookBot",
+  "meta-externalagent",
+  "Amazonbot",
+  "Applebot",
+  "DeepSeekBot",
+  "Bytespider",
+  "cohere-ai",
+  "CCBot",
+  "DiffBot",
+  "YouBot",
+  "FirecrawlAgent",
+  "Crawl4AI",
+  "BraveBot",
+  "SemrushBot",
+  "AhrefsBot",
+  "MJ12bot",
+  "DotBot",
+  "DataForSeoBot",
+  "Screaming Frog SEO Spider",
+  "SEOkicks",
+  "SEMrushBot",
+  "BLEXBot",
+  "Yandex",
+  "Baiduspider",
+  "Sogou",
+  "Exabot",
+  "facebookexternalhit",
+  "LinkedInBot",
+  "WhatsApp",
+  "Slackbot",
+  "TwitterBot",
+  "TelegramBot",
+  "Discordbot",
+  "PinterestBot",
+  "TumblrBot",
+  "ViberBot",
+  "SkypeUriPreview",
+  "redditbot",
+  "Snapchat",
+  "TikTok"
+];
+function generateRobotsTxt(config) {
+  const lines = [
+    "# robots.txt generated by aeo.js",
+    "# Allow AI crawlers to index this site",
+    "",
+    "# Traditional search engines",
+    "User-agent: Googlebot",
+    "Allow: /",
+    "",
+    "User-agent: Bingbot",
+    "Allow: /",
+    "",
+    "# AI crawlers and answer engines"
+  ];
+  for (const crawler of AI_CRAWLERS) {
+    lines.push(`User-agent: ${crawler}`);
+    lines.push("Allow: /");
+    lines.push("");
+  }
+  lines.push("# Default for all other bots");
+  lines.push("User-agent: *");
+  lines.push("Allow: /");
+  lines.push("");
+  if (config.url) {
+    lines.push(`Sitemap: ${config.url}/sitemap.xml`);
+  }
+  lines.push("");
+  lines.push("# AEO (Answer Engine Optimization) files");
+  lines.push("# These help LLMs understand your content better");
+  lines.push(`# ${config.url}/llms.txt`);
+  lines.push(`# ${config.url}/llms-full.txt`);
+  lines.push(`# ${config.url}/docs.json`);
+  lines.push(`# ${config.url}/ai-index.json`);
+  return lines.join("\n");
+}
+
+// src/core/detect.ts
+function detectFramework(projectRoot = process.cwd()) {
+  const packageJson = readPackageJson(projectRoot);
+  const dependencies = {
+    ...packageJson.dependencies,
+    ...packageJson.devDependencies
+  };
+  if (dependencies["next"]) {
+    return {
+      framework: "next",
+      contentDir: "app",
+      outDir: "out"
+    };
+  }
+  if (dependencies["nuxt"] || dependencies["@nuxt/kit"]) {
+    return {
+      framework: "nuxt",
+      contentDir: "content",
+      outDir: ".output/public"
+    };
+  }
+  if (dependencies["astro"] || dependencies["@astrojs/astro"]) {
+    return {
+      framework: "astro",
+      contentDir: "src/content",
+      outDir: "dist"
+    };
+  }
+  if (dependencies["@remix-run/dev"]) {
+    return {
+      framework: "remix",
+      contentDir: "app",
+      outDir: "build/client"
+    };
+  }
+  if (dependencies["@sveltejs/kit"]) {
+    return {
+      framework: "sveltekit",
+      contentDir: "src",
+      outDir: "build"
+    };
+  }
+  if (dependencies["@angular/core"]) {
+    return {
+      framework: "angular",
+      contentDir: "src",
+      outDir: "dist"
+    };
+  }
+  if (dependencies["@docusaurus/core"]) {
+    return {
+      framework: "docusaurus",
+      contentDir: "docs",
+      outDir: "build"
+    };
+  }
+  if (dependencies["vite"]) {
+    return {
+      framework: "vite",
+      contentDir: "src",
+      outDir: "dist"
+    };
+  }
+  return {
+    framework: "unknown",
+    contentDir: "src",
+    outDir: "dist"
+  };
+}
+function resolveConfig(config = {}) {
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x;
+  const frameworkInfo = detectFramework();
+  return {
+    title: config.title || "My Site",
+    description: config.description || "",
+    url: config.url || "https://example.com",
+    contentDir: config.contentDir || frameworkInfo.contentDir,
+    outDir: config.outDir || frameworkInfo.outDir,
+    pages: config.pages || [],
+    generators: {
+      robotsTxt: ((_a = config.generators) == null ? void 0 : _a.robotsTxt) !== false,
+      llmsTxt: ((_b = config.generators) == null ? void 0 : _b.llmsTxt) !== false,
+      llmsFullTxt: ((_c = config.generators) == null ? void 0 : _c.llmsFullTxt) !== false,
+      rawMarkdown: ((_d = config.generators) == null ? void 0 : _d.rawMarkdown) !== false,
+      manifest: ((_e = config.generators) == null ? void 0 : _e.manifest) !== false,
+      sitemap: ((_f = config.generators) == null ? void 0 : _f.sitemap) !== false,
+      aiIndex: ((_g = config.generators) == null ? void 0 : _g.aiIndex) !== false
+    },
+    robots: {
+      allow: ((_h = config.robots) == null ? void 0 : _h.allow) || ["/"],
+      disallow: ((_i = config.robots) == null ? void 0 : _i.disallow) || [],
+      crawlDelay: ((_j = config.robots) == null ? void 0 : _j.crawlDelay) || 0,
+      sitemap: ((_k = config.robots) == null ? void 0 : _k.sitemap) || ""
+    },
+    widget: {
+      enabled: ((_l = config.widget) == null ? void 0 : _l.enabled) !== false,
+      position: ((_m = config.widget) == null ? void 0 : _m.position) || "bottom-right",
+      theme: {
+        background: ((_o = (_n = config.widget) == null ? void 0 : _n.theme) == null ? void 0 : _o.background) || "rgba(18, 18, 24, 0.9)",
+        text: ((_q = (_p = config.widget) == null ? void 0 : _p.theme) == null ? void 0 : _q.text) || "#C0C0C5",
+        accent: ((_s = (_r = config.widget) == null ? void 0 : _r.theme) == null ? void 0 : _s.accent) || "#E8E8EA",
+        badge: ((_u = (_t = config.widget) == null ? void 0 : _t.theme) == null ? void 0 : _u.badge) || "#4ADE80"
+      },
+      humanLabel: ((_v = config.widget) == null ? void 0 : _v.humanLabel) || "Human",
+      aiLabel: ((_w = config.widget) == null ? void 0 : _w.aiLabel) || "AI",
+      showBadge: ((_x = config.widget) == null ? void 0 : _x.showBadge) !== false
+    }
+  };
+}
+function parseFrontmatter(content) {
+  const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)/);
+  if (frontmatterMatch) {
+    const frontmatterStr = frontmatterMatch[1];
+    const contentWithoutFrontmatter = frontmatterMatch[2];
+    const frontmatter = {};
+    const lines = frontmatterStr.split("\n");
+    for (const line of lines) {
+      const [key, ...valueParts] = line.split(":");
+      if (key && valueParts.length > 0) {
+        const value = valueParts.join(":").trim();
+        frontmatter[key.trim()] = value.replace(/^["']|["']$/g, "");
+      }
+    }
+    return { frontmatter, content: contentWithoutFrontmatter };
+  }
+  return { frontmatter: {}, content };
+}
+function bumpHeadings(content, levels = 1) {
+  return content.replace(/^(#{1,6})\s/gm, (match, hashes) => {
+    const newLevel = Math.min(hashes.length + levels, 6);
+    return "#".repeat(newLevel) + " ";
+  });
+}
+function extractTitle(content) {
+  const h1Match = content.match(/^#\s+(.+)$/m);
+  if (h1Match) return h1Match[1];
+  const h2Match = content.match(/^##\s+(.+)$/m);
+  if (h2Match) return h2Match[1];
+  const firstLine = content.split("\n")[0];
+  return firstLine.slice(0, 100);
+}
+function readPackageJson(projectRoot = process.cwd()) {
+  const packageJsonPath = join(projectRoot, "package.json");
+  if (!existsSync(packageJsonPath)) {
+    return {};
+  }
+  try {
+    const content = readFileSync(packageJsonPath, "utf-8");
+    return JSON.parse(content);
+  } catch {
+    return {};
+  }
+}
+
+// src/core/llms-txt.ts
+function collectMarkdownFiles(dir, base = dir) {
+  const files = [];
+  try {
+    const entries = readdirSync(dir);
+    for (const entry of entries) {
+      const fullPath = join(dir, entry);
+      const stat = statSync(fullPath);
+      if (stat.isDirectory() && !entry.startsWith(".") && entry !== "node_modules") {
+        files.push(...collectMarkdownFiles(fullPath, base));
+      } else if (stat.isFile() && (extname(entry) === ".md" || extname(entry) === ".mdx")) {
+        const content = readFileSync(fullPath, "utf-8");
+        const { frontmatter, content: mainContent } = parseFrontmatter(content);
+        const relativePath = relative(base, fullPath);
+        files.push({
+          path: relativePath,
+          content: mainContent,
+          title: frontmatter.title || extractTitle(mainContent),
+          description: frontmatter.description,
+          frontmatter
+        });
+      }
+    }
+  } catch (error) {
+    console.warn(`Warning: Could not read directory ${dir}:`, error);
+  }
+  return files;
+}
+function generateLlmsTxt(config) {
+  const lines = [
+    `# ${config.title}`,
+    ""
+  ];
+  if (config.description) {
+    lines.push(`> ${config.description}`);
+    lines.push("");
+  }
+  lines.push("## About");
+  lines.push("");
+  lines.push("This file provides a structured overview of the documentation and content available on this site,");
+  lines.push("optimized for consumption by Large Language Models (LLMs) and AI assistants.");
+  lines.push("");
+  if (config.pages && config.pages.length > 0) {
+    lines.push("## Pages");
+    lines.push("");
+    for (const page of config.pages) {
+      const url = `${config.url}${page.pathname === "/" ? "" : page.pathname}`;
+      const title = page.title || page.pathname;
+      lines.push(`- [${title}](${url})`);
+      if (page.description) {
+        lines.push(`  ${page.description}`);
+      }
+    }
+    lines.push("");
+  }
+  const markdownFiles = collectMarkdownFiles(config.contentDir);
+  if (markdownFiles.length > 0) {
+    lines.push("## Documentation");
+    lines.push("");
+    const grouped = {};
+    for (const file of markdownFiles) {
+      const dir = file.path.split("/")[0] || "root";
+      if (!grouped[dir]) grouped[dir] = [];
+      grouped[dir].push(file);
+    }
+    for (const [dir, files] of Object.entries(grouped)) {
+      lines.push(`### ${dir === "root" ? "Main Documentation" : dir}`);
+      lines.push("");
+      for (const file of files) {
+        const url = `${config.url}/${file.path.replace(/\.mdx?$/, "")}`;
+        lines.push(`- [${file.title}](${url})`);
+        if (file.description) {
+          lines.push(`  ${file.description}`);
+        }
+      }
+      lines.push("");
+    }
+  }
+  lines.push("## Quick Links");
+  lines.push("");
+  lines.push(`- Full Documentation: ${config.url}/llms-full.txt`);
+  lines.push(`- Documentation Manifest: ${config.url}/docs.json`);
+  lines.push(`- AI-Optimized Index: ${config.url}/ai-index.json`);
+  lines.push(`- Sitemap: ${config.url}/sitemap.xml`);
+  lines.push("");
+  lines.push("## For LLMs");
+  lines.push("");
+  lines.push("To get the complete documentation in a single file, request:");
+  lines.push(`${config.url}/llms-full.txt`);
+  lines.push("");
+  lines.push("For structured access to individual pages with metadata:");
+  lines.push(`${config.url}/docs.json`);
+  lines.push("");
+  lines.push("For RAG (Retrieval Augmented Generation) systems:");
+  lines.push(`${config.url}/ai-index.json`);
+  lines.push("");
+  lines.push("---");
+  lines.push("Generated by aeo.js - Answer Engine Optimization for the modern web");
+  lines.push("Learn more at https://aeojs.org");
+  return lines.join("\n");
+}
+function collectAndConcatenateMarkdown(dir, base = dir) {
+  const sections = [];
+  try {
+    const entries = readdirSync(dir).sort();
+    for (const entry of entries) {
+      const fullPath = join(dir, entry);
+      const stat = statSync(fullPath);
+      if (stat.isDirectory() && !entry.startsWith(".") && entry !== "node_modules") {
+        const subSections = collectAndConcatenateMarkdown(fullPath, base);
+        if (subSections.length > 0) {
+          sections.push(...subSections);
+        }
+      } else if (stat.isFile() && (extname(entry) === ".md" || extname(entry) === ".mdx")) {
+        const content = readFileSync(fullPath, "utf-8");
+        const { frontmatter, content: mainContent } = parseFrontmatter(content);
+        const relativePath = relative(base, fullPath);
+        const sectionLines = [
+          "---",
+          "",
+          `# ${frontmatter.title || relativePath}`,
+          "",
+          `Source: ${relativePath}`,
+          ""
+        ];
+        if (frontmatter.description) {
+          sectionLines.push(`> ${frontmatter.description}`);
+          sectionLines.push("");
+        }
+        const bumpedContent = bumpHeadings(mainContent, 1);
+        sectionLines.push(bumpedContent);
+        sectionLines.push("");
+        sections.push(sectionLines.join("\n"));
+      }
+    }
+  } catch (error) {
+    console.warn(`Warning: Could not read directory ${dir}:`, error);
+  }
+  return sections;
+}
+function generateLlmsFullTxt(config) {
+  const lines = [
+    `# ${config.title} - Complete Documentation`,
+    "",
+    `This file contains all documentation concatenated into a single file for easy consumption by LLMs.`,
+    ""
+  ];
+  if (config.description) {
+    lines.push(`> ${config.description}`);
+    lines.push("");
+  }
+  lines.push("## Table of Contents");
+  lines.push("");
+  lines.push("This document includes all content from this project.");
+  lines.push("Each section is separated by a horizontal rule (---) for easy parsing.");
+  lines.push("");
+  let hasContent = false;
+  if (config.pages && config.pages.length > 0) {
+    for (const page of config.pages) {
+      const url = `${config.url}${page.pathname === "/" ? "" : page.pathname}`;
+      const title = page.title || page.pathname;
+      const sectionLines = [
+        "---",
+        "",
+        `# ${title}`,
+        "",
+        `URL: ${url}`,
+        ""
+      ];
+      if (page.description) {
+        sectionLines.push(`> ${page.description}`);
+        sectionLines.push("");
+      }
+      if (page.content) {
+        sectionLines.push(page.content);
+        sectionLines.push("");
+      }
+      lines.push(sectionLines.join("\n"));
+      hasContent = true;
+    }
+  }
+  const sections = collectAndConcatenateMarkdown(config.contentDir);
+  if (sections.length > 0) {
+    lines.push(...sections);
+    hasContent = true;
+  }
+  if (!hasContent) {
+    lines.push("---");
+    lines.push("");
+    lines.push(`# ${config.title}`);
+    lines.push("");
+    lines.push(`URL: ${config.url}`);
+    lines.push("");
+    if (config.description) {
+      lines.push(config.description);
+      lines.push("");
+    }
+  }
+  lines.push("---");
+  lines.push("");
+  lines.push("## About This Document");
+  lines.push("");
+  lines.push("This concatenated documentation file is generated automatically by aeo.js");
+  lines.push("to make it easier for AI systems to understand the complete context of this project.");
+  lines.push("");
+  lines.push(`For a structured index, see: ${config.url}/llms.txt`);
+  lines.push(`For individual files, see: ${config.url}/docs.json`);
+  lines.push("");
+  lines.push("Generated by aeo.js - https://aeojs.org");
+  return lines.join("\n");
+}
+function ensureDir(path) {
+  mkdirSync(path, { recursive: true });
+}
+function copyMarkdownFiles(config) {
+  const copiedFiles = [];
+  function copyRecursive(dir, base = config.contentDir) {
+    try {
+      const entries = readdirSync(dir);
+      for (const entry of entries) {
+        const fullPath = join(dir, entry);
+        const stat = statSync(fullPath);
+        if (stat.isDirectory() && !entry.startsWith(".") && entry !== "node_modules") {
+          copyRecursive(fullPath, base);
+        } else if (stat.isFile() && extname(entry) === ".md") {
+          const relativePath = relative(base, fullPath);
+          const destPath = join(config.outDir, relativePath);
+          ensureDir(dirname(destPath));
+          try {
+            copyFileSync(fullPath, destPath);
+            copiedFiles.push({
+              source: fullPath,
+              destination: destPath
+            });
+          } catch (error) {
+            console.warn(`Warning: Could not copy ${fullPath}:`, error);
+          }
+        }
+      }
+    } catch (error) {
+      console.warn(`Warning: Could not read directory ${dir}:`, error);
+    }
+  }
+  copyRecursive(config.contentDir);
+  return copiedFiles;
+}
+function generatePageMarkdownFiles(config) {
+  const generated = [];
+  const pages = config.pages || [];
+  for (const page of pages) {
+    const pageTitle = page.title || (page.pathname === "/" ? config.title : void 0);
+    if (!page.content && !pageTitle) continue;
+    let filename;
+    if (page.pathname === "/") {
+      filename = "index.md";
+    } else {
+      const clean = page.pathname.replace(/^\//, "").replace(/\/$/, "");
+      filename = clean.includes("/") ? `${clean}.md` : `${clean}.md`;
+    }
+    const destPath = join(config.outDir, filename);
+    const pageUrl = page.pathname === "/" ? config.url : `${config.url.replace(/\/$/, "")}${page.pathname}`;
+    const lines = [];
+    lines.push("---");
+    if (pageTitle) lines.push(`title: "${pageTitle}"`);
+    if (page.description) lines.push(`description: "${page.description}"`);
+    lines.push(`url: ${pageUrl}`);
+    lines.push(`source: ${pageUrl}`);
+    lines.push(`generated_by: aeo.js`);
+    lines.push("---", "");
+    if (pageTitle) {
+      lines.push(`# ${pageTitle}`, "");
+    }
+    if (page.description) {
+      lines.push(`${page.description}`, "");
+    }
+    if (page.content) {
+      lines.push(page.content);
+    }
+    const content = lines.join("\n");
+    ensureDir(dirname(destPath));
+    try {
+      writeFileSync(destPath, content, "utf-8");
+      generated.push({ pathname: page.pathname, destination: destPath });
+    } catch {
+    }
+  }
+  return generated;
+}
+function collectManifestEntries(dir, config, base = dir) {
+  const entries = [];
+  try {
+    const files = readdirSync(dir);
+    for (const file of files) {
+      const fullPath = join(dir, file);
+      const stat = statSync(fullPath);
+      if (stat.isDirectory() && !file.startsWith(".") && file !== "node_modules") {
+        entries.push(...collectManifestEntries(fullPath, config, base));
+      } else if (stat.isFile() && (extname(file) === ".md" || extname(file) === ".mdx")) {
+        const content = readFileSync(fullPath, "utf-8");
+        const { frontmatter, content: mainContent } = parseFrontmatter(content);
+        const relativePath = relative(base, fullPath);
+        const urlPath = relativePath.replace(/\.mdx?$/, "");
+        entries.push({
+          url: `${config.url}/${urlPath}`,
+          title: frontmatter.title || extractTitle(mainContent),
+          description: frontmatter.description,
+          lastModified: stat.mtime.toISOString()
+        });
+      }
+    }
+  } catch (error) {
+    console.warn(`Warning: Could not read directory ${dir}:`, error);
+  }
+  return entries;
+}
+function generateManifest(config) {
+  const entries = [];
+  if (config.pages && config.pages.length > 0) {
+    for (const page of config.pages) {
+      entries.push({
+        url: `${config.url}${page.pathname === "/" ? "" : page.pathname}`,
+        title: page.title || page.pathname,
+        description: page.description
+      });
+    }
+  }
+  entries.push(...collectManifestEntries(config.contentDir, config));
+  const manifest = {
+    version: "1.0",
+    generated: (/* @__PURE__ */ new Date()).toISOString(),
+    site: {
+      title: config.title,
+      description: config.description,
+      url: config.url
+    },
+    documents: entries.sort((a, b) => a.url.localeCompare(b.url)),
+    metadata: {
+      totalDocuments: entries.length,
+      generator: "aeo.js",
+      generatorUrl: "https://aeojs.org"
+    }
+  };
+  return JSON.stringify(manifest, null, 2);
+}
+function collectUrls(dir, config, base = dir) {
+  const urls = [];
+  try {
+    const entries = readdirSync(dir);
+    for (const entry of entries) {
+      const fullPath = join(dir, entry);
+      const stat = statSync(fullPath);
+      if (stat.isDirectory() && !entry.startsWith(".") && entry !== "node_modules") {
+        urls.push(...collectUrls(fullPath, config, base));
+      } else if (stat.isFile() && (extname(entry) === ".md" || extname(entry) === ".mdx" || extname(entry) === ".html")) {
+        const relativePath = relative(base, fullPath);
+        const urlPath = relativePath.replace(/\.(md|mdx|html)$/, "");
+        urls.push(`${config.url}/${urlPath}`);
+      }
+    }
+  } catch (error) {
+    console.warn(`Warning: Could not read directory ${dir}:`, error);
+  }
+  return urls;
+}
+function escapeXml(str) {
+  return str.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+function generateSitemap(config) {
+  const urls = [];
+  if (config.pages && config.pages.length > 0) {
+    for (const page of config.pages) {
+      urls.push(`${config.url}${page.pathname === "/" ? "" : page.pathname}`);
+    }
+  }
+  if (config.contentDir) {
+    urls.push(...collectUrls(config.contentDir, config));
+  }
+  const lines = [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+  ];
+  urls.push(config.url);
+  const uniqueUrls = [...new Set(urls)].sort();
+  for (const url of uniqueUrls) {
+    lines.push("  <url>");
+    lines.push(`    <loc>${escapeXml(url)}</loc>`);
+    lines.push(`    <lastmod>${(/* @__PURE__ */ new Date()).toISOString().split("T")[0]}</lastmod>`);
+    lines.push("    <changefreq>weekly</changefreq>");
+    lines.push("    <priority>0.8</priority>");
+    lines.push("  </url>");
+  }
+  lines.push("</urlset>");
+  return lines.join("\n");
+}
+function extractKeywords(content) {
+  const words = content.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter((word) => word.length > 3);
+  const wordCount = {};
+  for (const word of words) {
+    wordCount[word] = (wordCount[word] || 0) + 1;
+  }
+  return Object.entries(wordCount).sort((a, b) => b[1] - a[1]).slice(0, 10).map(([word]) => word);
+}
+function chunkContent(content, maxLength = 2e3) {
+  const chunks = [];
+  const paragraphs = content.split("\n\n");
+  let currentChunk = "";
+  for (const paragraph of paragraphs) {
+    if (currentChunk.length + paragraph.length > maxLength && currentChunk.length > 0) {
+      chunks.push(currentChunk.trim());
+      currentChunk = "";
+    }
+    currentChunk += paragraph + "\n\n";
+  }
+  if (currentChunk.trim()) {
+    chunks.push(currentChunk.trim());
+  }
+  return chunks;
+}
+function collectAIIndexEntries(dir, config, base = dir) {
+  const entries = [];
+  try {
+    const files = readdirSync(dir);
+    for (const file of files) {
+      const fullPath = join(dir, file);
+      const stat = statSync(fullPath);
+      if (stat.isDirectory() && !file.startsWith(".") && file !== "node_modules") {
+        entries.push(...collectAIIndexEntries(fullPath, config, base));
+      } else if (stat.isFile() && (extname(file) === ".md" || extname(file) === ".mdx")) {
+        const content = readFileSync(fullPath, "utf-8");
+        const { frontmatter, content: mainContent } = parseFrontmatter(content);
+        const relativePath = relative(base, fullPath);
+        const urlPath = relativePath.replace(/\.mdx?$/, "");
+        const url = `${config.url}/${urlPath}`;
+        const chunks = chunkContent(mainContent);
+        const title = frontmatter.title || extractTitle(mainContent);
+        const keywords = extractKeywords(mainContent);
+        chunks.forEach((chunk, index) => {
+          const id = createHash("sha256").update(`${url}-${index}`).digest("hex").slice(0, 16);
+          entries.push({
+            id,
+            url,
+            title: chunks.length > 1 ? `${title} (Part ${index + 1})` : title,
+            content: chunk,
+            description: frontmatter.description,
+            keywords,
+            metadata: {
+              ...frontmatter,
+              chunkIndex: index,
+              totalChunks: chunks.length,
+              sourcePath: relativePath
+            }
+          });
+        });
+      }
+    }
+  } catch (error) {
+    console.warn(`Warning: Could not read directory ${dir}:`, error);
+  }
+  return entries;
+}
+function generateAIIndex(config) {
+  const entries = [];
+  if (config.pages && config.pages.length > 0) {
+    for (const page of config.pages) {
+      const url = `${config.url}${page.pathname === "/" ? "" : page.pathname}`;
+      const title = page.title || page.pathname;
+      const content = page.content || "";
+      if (content) {
+        const chunks = chunkContent(content);
+        const keywords = extractKeywords(content);
+        chunks.forEach((chunk, index2) => {
+          const id = createHash("sha256").update(`${url}-${index2}`).digest("hex").slice(0, 16);
+          entries.push({
+            id,
+            url,
+            title: chunks.length > 1 ? `${title} (Part ${index2 + 1})` : title,
+            content: chunk,
+            description: page.description,
+            keywords,
+            metadata: {
+              chunkIndex: index2,
+              totalChunks: chunks.length,
+              sourcePath: page.pathname
+            }
+          });
+        });
+      } else {
+        const id = createHash("sha256").update(url).digest("hex").slice(0, 16);
+        entries.push({
+          id,
+          url,
+          title,
+          content: page.description || title,
+          description: page.description,
+          keywords: []
+        });
+      }
+    }
+  }
+  entries.push(...collectAIIndexEntries(config.contentDir, config));
+  const index = {
+    version: "1.0",
+    generated: (/* @__PURE__ */ new Date()).toISOString(),
+    site: {
+      title: config.title,
+      description: config.description,
+      url: config.url
+    },
+    entries: entries.sort((a, b) => a.id.localeCompare(b.id)),
+    metadata: {
+      totalEntries: entries.length,
+      generator: "aeo.js",
+      generatorUrl: "https://aeojs.org",
+      embedding: {
+        recommended: "text-embedding-ada-002",
+        dimensions: 1536
+      }
+    }
+  };
+  return JSON.stringify(index, null, 2);
+}
+async function generateAEOFiles(configOrRoot, maybeConfig) {
+  var _a;
+  let config;
+  if (typeof configOrRoot === "string") {
+    config = resolveConfig({ ...maybeConfig, outDir: configOrRoot });
+  } else if (configOrRoot && typeof configOrRoot === "object" && "generators" in configOrRoot && typeof ((_a = configOrRoot.generators) == null ? void 0 : _a.robotsTxt) === "boolean") {
+    config = configOrRoot;
+  } else {
+    config = resolveConfig(configOrRoot);
+  }
+  const outDir = config.outDir;
+  const files = [];
+  const errors = [];
+  if (!existsSync(outDir)) {
+    mkdirSync(outDir, { recursive: true });
+  }
+  if (config.generators.robotsTxt) {
+    try {
+      const content = generateRobotsTxt(config);
+      writeFileSync(join(outDir, "robots.txt"), content, "utf-8");
+      files.push("robots.txt");
+    } catch (e) {
+      errors.push(`robots.txt: ${e.message}`);
+    }
+  }
+  if (config.generators.llmsTxt) {
+    try {
+      const content = generateLlmsTxt(config);
+      writeFileSync(join(outDir, "llms.txt"), "\uFEFF" + content, "utf-8");
+      files.push("llms.txt");
+    } catch (e) {
+      errors.push(`llms.txt: ${e.message}`);
+    }
+  }
+  if (config.generators.llmsFullTxt) {
+    try {
+      const content = generateLlmsFullTxt(config);
+      writeFileSync(join(outDir, "llms-full.txt"), "\uFEFF" + content, "utf-8");
+      files.push("llms-full.txt");
+    } catch (e) {
+      errors.push(`llms-full.txt: ${e.message}`);
+    }
+  }
+  if (config.generators.rawMarkdown) {
+    try {
+      const generated = generatePageMarkdownFiles(config);
+      for (const f of generated) {
+        files.push(f.destination);
+      }
+    } catch (e) {
+      errors.push(`page-markdown: ${e.message}`);
+    }
+    try {
+      const copied = copyMarkdownFiles(config);
+      for (const f of copied) {
+        files.push(f.destination);
+      }
+    } catch (e) {
+      errors.push(`raw-markdown: ${e.message}`);
+    }
+  }
+  if (config.generators.manifest) {
+    try {
+      const content = generateManifest(config);
+      writeFileSync(join(outDir, "docs.json"), content, "utf-8");
+      files.push("docs.json");
+    } catch (e) {
+      errors.push(`docs.json: ${e.message}`);
+    }
+  }
+  if (config.generators.sitemap) {
+    try {
+      const content = generateSitemap(config);
+      writeFileSync(join(outDir, "sitemap.xml"), content, "utf-8");
+      files.push("sitemap.xml");
+    } catch (e) {
+      errors.push(`sitemap.xml: ${e.message}`);
+    }
+  }
+  if (config.generators.aiIndex) {
+    try {
+      const content = generateAIIndex(config);
+      writeFileSync(join(outDir, "ai-index.json"), content, "utf-8");
+      files.push("ai-index.json");
+    } catch (e) {
+      errors.push(`ai-index.json: ${e.message}`);
+    }
+  }
+  return { files, errors };
+}
+function scanNextPages(projectRoot) {
+  const pages = [];
+  for (const base of ["app", "src/app"]) {
+    const dir = join(projectRoot, base);
+    if (existsSync(dir)) scanAppRouter(dir, dir, pages);
+  }
+  for (const base of ["pages", "src/pages"]) {
+    const dir = join(projectRoot, base);
+    if (existsSync(dir)) scanPagesRouter(dir, dir, pages);
+  }
+  return pages;
+}
+function scanAppRouter(dir, base, pages) {
+  try {
+    const entries = readdirSync(dir);
+    for (const entry of entries) {
+      const fullPath = join(dir, entry);
+      const stat = statSync(fullPath);
+      if (stat.isDirectory() && !entry.startsWith(".") && !entry.startsWith("_") && !entry.startsWith("(") && entry !== "api") {
+        scanAppRouter(fullPath, base, pages);
+      } else if (entry.match(/^page\.(tsx?|jsx?|mdx?)$/)) {
+        const relative8 = dir.slice(base.length);
+        const pathname = relative8 || "/";
+        const name = pathname.split("/").filter(Boolean).pop();
+        pages.push({
+          pathname,
+          title: name ? name.charAt(0).toUpperCase() + name.slice(1) : void 0
+        });
+      }
+    }
+  } catch {
+  }
+}
+function scanPagesRouter(dir, base, pages) {
+  try {
+    const entries = readdirSync(dir);
+    for (const entry of entries) {
+      const fullPath = join(dir, entry);
+      const stat = statSync(fullPath);
+      if (stat.isDirectory() && !entry.startsWith(".") && !entry.startsWith("_") && entry !== "api") {
+        scanPagesRouter(fullPath, base, pages);
+      } else if (entry.match(/\.(tsx?|jsx?|mdx?)$/) && !entry.startsWith("_") && !entry.startsWith("[")) {
+        const relative8 = fullPath.slice(base.length);
+        let pathname = relative8.replace(/\.(tsx?|jsx?|mdx?)$/, "");
+        if (pathname.endsWith("/index")) pathname = pathname.slice(0, -6) || "/";
+        pathname = pathname.replace(/\/+/g, "/") || "/";
+        const name = entry.replace(/\.(tsx?|jsx?|mdx?)$/, "");
+        pages.push({
+          pathname,
+          title: name === "index" ? void 0 : name.charAt(0).toUpperCase() + name.slice(1)
+        });
+      }
+    }
+  } catch {
+  }
+}
+function withAeo(nextConfig = {}) {
+  const { aeo: aeoOptions = {}, ...restConfig } = nextConfig;
+  return {
+    ...restConfig,
+    webpack(config, options) {
+      if (typeof nextConfig.webpack === "function") {
+        config = nextConfig.webpack(config, options);
+      }
+      if (!options.isServer && !options.dev) {
+        const projectRoot = process.cwd();
+        const discoveredPages = scanNextPages(projectRoot);
+        for (const page of discoveredPages) {
+          if (page.pathname === "/" && !page.title) {
+            page.title = aeoOptions.title;
+          }
+          if (!page.description && aeoOptions.description) {
+            page.description = aeoOptions.description;
+          }
+        }
+        const contentDir = aeoOptions.contentDir || (existsSync(join(projectRoot, "src")) ? join(projectRoot, "src") : projectRoot);
+        const resolvedConfig = resolveConfig({
+          ...aeoOptions,
+          outDir: aeoOptions.outDir || join(projectRoot, "public"),
+          contentDir,
+          pages: [...aeoOptions.pages || [], ...discoveredPages]
+        });
+        if (!existsSync(resolvedConfig.outDir)) {
+          mkdirSync(resolvedConfig.outDir, { recursive: true });
+        }
+        config.plugins.push({
+          apply: (compiler) => {
+            compiler.hooks.afterEmit.tapAsync("AeoPlugin", async (_compilation, callback) => {
+              console.log("[aeo.js] Generating AEO files for Next.js...");
+              try {
+                const result = await generateAEOFiles(resolvedConfig);
+                if (result.files.length > 0) {
+                  console.log(`[aeo.js] Generated ${result.files.length} files`);
+                }
+                if (result.errors.length > 0) {
+                  console.error("[aeo.js] Errors:", result.errors);
+                }
+              } catch (error) {
+                console.error("[aeo.js] Failed to generate AEO files:", error);
+              }
+              callback();
+            });
+          }
+        });
+      }
+      return config;
+    }
+  };
+}
+async function generateAeoMetadata(config) {
+  const resolvedConfig = resolveConfig(config);
+  if (process.env.NODE_ENV === "production") {
+    await generateAEOFiles(resolvedConfig);
+  }
+  return {
+    title: resolvedConfig.title,
+    description: resolvedConfig.description,
+    alternates: {
+      types: {
+        "text/plain": [
+          { url: "/llms.txt", title: "LLM Summary" },
+          { url: "/llms-full.txt", title: "Full Content for LLMs" }
+        ],
+        "application/json": [
+          { url: "/docs.json", title: "Documentation Manifest" },
+          { url: "/ai-index.json", title: "AI-Optimized Index" }
+        ]
+      }
+    }
+  };
+}
+function extractText(html) {
+  let text = html;
+  text = text.replace(/<script[\s\S]*?<\/script>/gi, "");
+  text = text.replace(/<style[\s\S]*?<\/style>/gi, "");
+  text = text.replace(/<svg[\s\S]*?<\/svg>/gi, "");
+  const mainMatch = text.match(/<main[^>]*>([\s\S]*)<\/main>/i);
+  if (mainMatch) {
+    text = mainMatch[1];
+  } else {
+    text = text.replace(/<nav[\s\S]*?<\/nav>/gi, "");
+    text = text.replace(/<header[\s\S]*?<\/header>/gi, "");
+    text = text.replace(/<footer[\s\S]*?<\/footer>/gi, "");
+  }
+  text = text.replace(/<a[^>]+href=["']([^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi, (_, url, inner) => {
+    if (/<(?:h[1-6]|div|p|section)[^>]*>/i.test(inner)) {
+      const cleanInner = inner.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+      return `
+[${cleanInner.slice(0, 120).trim()}](${url})
+`;
+    }
+    return `[${inner}](${url})`;
+  });
+  text = text.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, "\n\n## $1\n\n");
+  text = text.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, "\n\n## $1\n\n");
+  text = text.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, "\n\n### $1\n\n");
+  text = text.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, "\n\n#### $1\n\n");
+  text = text.replace(/<h5[^>]*>([\s\S]*?)<\/h5>/gi, "\n\n##### $1\n\n");
+  text = text.replace(/<h6[^>]*>([\s\S]*?)<\/h6>/gi, "\n\n###### $1\n\n");
+  text = text.replace(/<a[^>]+href=["']([^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi, "[$2]($1)");
+  text = text.replace(/<(?:strong|b)[^>]*>([\s\S]*?)<\/(?:strong|b)>/gi, "**$1**");
+  text = text.replace(/<(?:em|i)[^>]*>([\s\S]*?)<\/(?:em|i)>/gi, "*$1*");
+  text = text.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, "\n- $1");
+  text = text.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, "\n\n> $1\n\n");
+  text = text.replace(/<hr[^>]*\/?>/gi, "\n\n---\n\n");
+  text = text.replace(/<br[^>]*\/?>/gi, "\n");
+  text = text.replace(/<\/p>/gi, "\n\n");
+  text = text.replace(/<p[^>]*>/gi, "");
+  text = text.replace(/<\/?(?:div|section|article|header|main|aside|figure|figcaption|table|thead|tbody|tr|td|th|ul|ol|dl|dt|dd)[^>]*>/gi, "\n");
+  text = text.replace(/<[^>]+>/g, "");
+  text = text.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, " ").replace(/&copy;/g, "(c)");
+  text = text.replace(/[\u{1F1E0}-\u{1FAFF}\u{2600}-\u{27BF}\u{FE00}-\u{FE0F}\u{200D}\u{20E3}]/gu, "");
+  text = text.split("\n").map((l) => l.replace(/\s+/g, " ").trim()).join("\n");
+  text = text.replace(/\n{3,}/g, "\n\n");
+  text = text.replace(/\[[\s\n]+/g, "[").replace(/[\s\n]+\]/g, "]");
+  text = text.replace(/(#{2,6})\s*\n+\s*/g, "$1 ");
+  text = text.replace(/^#{2,6}\s*$/gm, "");
+  text = text.replace(/\n{3,}/g, "\n\n");
+  return text.trim().slice(0, 8e3);
+}
+function scanNextBuildOutput(projectRoot) {
+  const pages = [];
+  const serverAppDir = join(projectRoot, ".next", "server", "app");
+  if (!existsSync(serverAppDir)) return pages;
+  function walk(dir, basePath = "") {
+    var _a, _b;
+    try {
+      const entries = readdirSync(dir);
+      for (const entry of entries) {
+        const fullPath = join(dir, entry);
+        const stat = statSync(fullPath);
+        if (stat.isDirectory() && !entry.startsWith("_") && !entry.startsWith(".")) {
+          walk(fullPath, `${basePath}/${entry}`);
+        } else if (entry === "index.html") {
+          const html = readFileSync(fullPath, "utf-8");
+          const titleMatch = html.match(/<title>([^<]*)<\/title>/i);
+          const descMatch = html.match(/<meta\s+name=["']description["']\s+content=["']([^"']*)["']/i);
+          const textContent = extractText(html);
+          const pathname = basePath || "/";
+          pages.push({
+            pathname,
+            title: (_b = (_a = titleMatch == null ? void 0 : titleMatch[1]) == null ? void 0 : _a.split("|")[0]) == null ? void 0 : _b.trim(),
+            description: descMatch == null ? void 0 : descMatch[1],
+            content: textContent
+          });
+        }
+      }
+    } catch {
+    }
+  }
+  walk(serverAppDir);
+  return pages;
+}
+async function postBuild(config = {}) {
+  const projectRoot = process.cwd();
+  const discoveredPages = scanNextBuildOutput(projectRoot);
+  if (discoveredPages.length > 0) {
+    console.log(`[aeo.js] Discovered ${discoveredPages.length} pages from Next.js build output`);
+  }
+  for (const page of discoveredPages) {
+    if (page.pathname === "/" && !page.title && config.title) {
+      page.title = config.title;
+    }
+    if (!page.description && config.description) {
+      page.description = config.description;
+    }
+  }
+  const contentDir = config.contentDir || (existsSync(join(projectRoot, "src")) ? join(projectRoot, "src") : projectRoot);
+  const resolvedConfig = resolveConfig({
+    ...config,
+    outDir: config.outDir || join(projectRoot, "public"),
+    contentDir,
+    pages: [...config.pages || [], ...discoveredPages]
+  });
+  const result = await generateAEOFiles(resolvedConfig);
+  if (result.files.length > 0) {
+    console.log(`[aeo.js] Generated ${result.files.length} files`);
+  }
+  if (result.errors.length > 0) {
+    console.error("[aeo.js] Errors:", result.errors);
+  }
+}
+var next_default = withAeo;
+
+export { next_default as default, generateAeoMetadata, postBuild, withAeo };
+//# sourceMappingURL=next.mjs.map
+//# sourceMappingURL=next.mjs.map