aeo.js 0.0.1 → 0.0.2

package/dist/astro.js

@@ -0,0 +1,1179 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+var fs = require('fs');
+var path = require('path');
+require('minimatch');
+var crypto = require('crypto');
+
+// src/core/robots.ts
+var AI_CRAWLERS = [
+  "GPTBot",
+  "OAI-SearchBot",
+  "ChatGPT-User",
+  "ClaudeBot",
+  "Claude-Web",
+  "anthropic-ai",
+  "PerplexityBot",
+  "Google-Extended",
+  "Gemini-Deep-Research",
+  "Bingbot",
+  "FacebookBot",
+  "meta-externalagent",
+  "Amazonbot",
+  "Applebot",
+  "DeepSeekBot",
+  "Bytespider",
+  "cohere-ai",
+  "CCBot",
+  "DiffBot",
+  "YouBot",
+  "FirecrawlAgent",
+  "Crawl4AI",
+  "BraveBot",
+  "SemrushBot",
+  "AhrefsBot",
+  "MJ12bot",
+  "DotBot",
+  "DataForSeoBot",
+  "Screaming Frog SEO Spider",
+  "SEOkicks",
+  "SEMrushBot",
+  "BLEXBot",
+  "Yandex",
+  "Baiduspider",
+  "Sogou",
+  "Exabot",
+  "facebookexternalhit",
+  "LinkedInBot",
+  "WhatsApp",
+  "Slackbot",
+  "TwitterBot",
+  "TelegramBot",
+  "Discordbot",
+  "PinterestBot",
+  "TumblrBot",
+  "ViberBot",
+  "SkypeUriPreview",
+  "redditbot",
+  "Snapchat",
+  "TikTok"
+];
+function generateRobotsTxt(config) {
+  const lines = [
+    "# robots.txt generated by aeo.js",
+    "# Allow AI crawlers to index this site",
+    "",
+    "# Traditional search engines",
+    "User-agent: Googlebot",
+    "Allow: /",
+    "",
+    "User-agent: Bingbot",
+    "Allow: /",
+    "",
+    "# AI crawlers and answer engines"
+  ];
+  for (const crawler of AI_CRAWLERS) {
+    lines.push(`User-agent: ${crawler}`);
+    lines.push("Allow: /");
+    lines.push("");
+  }
+  lines.push("# Default for all other bots");
+  lines.push("User-agent: *");
+  lines.push("Allow: /");
+  lines.push("");
+  if (config.url) {
+    lines.push(`Sitemap: ${config.url}/sitemap.xml`);
+  }
+  lines.push("");
+  lines.push("# AEO (Answer Engine Optimization) files");
+  lines.push("# These help LLMs understand your content better");
+  lines.push(`# ${config.url}/llms.txt`);
+  lines.push(`# ${config.url}/llms-full.txt`);
+  lines.push(`# ${config.url}/docs.json`);
+  lines.push(`# ${config.url}/ai-index.json`);
+  return lines.join("\n");
+}
+
+// src/core/detect.ts
+function detectFramework(projectRoot = process.cwd()) {
+  const packageJson = readPackageJson(projectRoot);
+  const dependencies = {
+    ...packageJson.dependencies,
+    ...packageJson.devDependencies
+  };
+  if (dependencies["next"]) {
+    return {
+      framework: "next",
+      contentDir: "app",
+      outDir: "out"
+    };
+  }
+  if (dependencies["nuxt"] || dependencies["@nuxt/kit"]) {
+    return {
+      framework: "nuxt",
+      contentDir: "content",
+      outDir: ".output/public"
+    };
+  }
+  if (dependencies["astro"] || dependencies["@astrojs/astro"]) {
+    return {
+      framework: "astro",
+      contentDir: "src/content",
+      outDir: "dist"
+    };
+  }
+  if (dependencies["@remix-run/dev"]) {
+    return {
+      framework: "remix",
+      contentDir: "app",
+      outDir: "build/client"
+    };
+  }
+  if (dependencies["@sveltejs/kit"]) {
+    return {
+      framework: "sveltekit",
+      contentDir: "src",
+      outDir: "build"
+    };
+  }
+  if (dependencies["@angular/core"]) {
+    return {
+      framework: "angular",
+      contentDir: "src",
+      outDir: "dist"
+    };
+  }
+  if (dependencies["@docusaurus/core"]) {
+    return {
+      framework: "docusaurus",
+      contentDir: "docs",
+      outDir: "build"
+    };
+  }
+  if (dependencies["vite"]) {
+    return {
+      framework: "vite",
+      contentDir: "src",
+      outDir: "dist"
+    };
+  }
+  return {
+    framework: "unknown",
+    contentDir: "src",
+    outDir: "dist"
+  };
+}
+function resolveConfig(config = {}) {
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x;
+  const frameworkInfo = detectFramework();
+  return {
+    title: config.title || "My Site",
+    description: config.description || "",
+    url: config.url || "https://example.com",
+    contentDir: config.contentDir || frameworkInfo.contentDir,
+    outDir: config.outDir || frameworkInfo.outDir,
+    pages: config.pages || [],
+    generators: {
+      robotsTxt: ((_a = config.generators) == null ? void 0 : _a.robotsTxt) !== false,
+      llmsTxt: ((_b = config.generators) == null ? void 0 : _b.llmsTxt) !== false,
+      llmsFullTxt: ((_c = config.generators) == null ? void 0 : _c.llmsFullTxt) !== false,
+      rawMarkdown: ((_d = config.generators) == null ? void 0 : _d.rawMarkdown) !== false,
+      manifest: ((_e = config.generators) == null ? void 0 : _e.manifest) !== false,
+      sitemap: ((_f = config.generators) == null ? void 0 : _f.sitemap) !== false,
+      aiIndex: ((_g = config.generators) == null ? void 0 : _g.aiIndex) !== false
+    },
+    robots: {
+      allow: ((_h = config.robots) == null ? void 0 : _h.allow) || ["/"],
+      disallow: ((_i = config.robots) == null ? void 0 : _i.disallow) || [],
+      crawlDelay: ((_j = config.robots) == null ? void 0 : _j.crawlDelay) || 0,
+      sitemap: ((_k = config.robots) == null ? void 0 : _k.sitemap) || ""
+    },
+    widget: {
+      enabled: ((_l = config.widget) == null ? void 0 : _l.enabled) !== false,
+      position: ((_m = config.widget) == null ? void 0 : _m.position) || "bottom-right",
+      theme: {
+        background: ((_o = (_n = config.widget) == null ? void 0 : _n.theme) == null ? void 0 : _o.background) || "rgba(18, 18, 24, 0.9)",
+        text: ((_q = (_p = config.widget) == null ? void 0 : _p.theme) == null ? void 0 : _q.text) || "#C0C0C5",
+        accent: ((_s = (_r = config.widget) == null ? void 0 : _r.theme) == null ? void 0 : _s.accent) || "#E8E8EA",
+        badge: ((_u = (_t = config.widget) == null ? void 0 : _t.theme) == null ? void 0 : _u.badge) || "#4ADE80"
+      },
+      humanLabel: ((_v = config.widget) == null ? void 0 : _v.humanLabel) || "Human",
+      aiLabel: ((_w = config.widget) == null ? void 0 : _w.aiLabel) || "AI",
+      showBadge: ((_x = config.widget) == null ? void 0 : _x.showBadge) !== false
+    }
+  };
+}
+function parseFrontmatter(content) {
+  const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)/);
+  if (frontmatterMatch) {
+    const frontmatterStr = frontmatterMatch[1];
+    const contentWithoutFrontmatter = frontmatterMatch[2];
+    const frontmatter = {};
+    const lines = frontmatterStr.split("\n");
+    for (const line of lines) {
+      const [key, ...valueParts] = line.split(":");
+      if (key && valueParts.length > 0) {
+        const value = valueParts.join(":").trim();
+        frontmatter[key.trim()] = value.replace(/^["']|["']$/g, "");
+      }
+    }
+    return { frontmatter, content: contentWithoutFrontmatter };
+  }
+  return { frontmatter: {}, content };
+}
+function bumpHeadings(content, levels = 1) {
+  return content.replace(/^(#{1,6})\s/gm, (match, hashes) => {
+    const newLevel = Math.min(hashes.length + levels, 6);
+    return "#".repeat(newLevel) + " ";
+  });
+}
+function extractTitle(content) {
+  const h1Match = content.match(/^#\s+(.+)$/m);
+  if (h1Match) return h1Match[1];
+  const h2Match = content.match(/^##\s+(.+)$/m);
+  if (h2Match) return h2Match[1];
+  const firstLine = content.split("\n")[0];
+  return firstLine.slice(0, 100);
+}
+function readPackageJson(projectRoot = process.cwd()) {
+  const packageJsonPath = path.join(projectRoot, "package.json");
+  if (!fs.existsSync(packageJsonPath)) {
+    return {};
+  }
+  try {
+    const content = fs.readFileSync(packageJsonPath, "utf-8");
+    return JSON.parse(content);
+  } catch {
+    return {};
+  }
+}
+
+// src/core/llms-txt.ts
+function collectMarkdownFiles(dir, base = dir) {
+  const files = [];
+  try {
+    const entries = fs.readdirSync(dir);
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry);
+      const stat = fs.statSync(fullPath);
+      if (stat.isDirectory() && !entry.startsWith(".") && entry !== "node_modules") {
+        files.push(...collectMarkdownFiles(fullPath, base));
+      } else if (stat.isFile() && (path.extname(entry) === ".md" || path.extname(entry) === ".mdx")) {
+        const content = fs.readFileSync(fullPath, "utf-8");
+        const { frontmatter, content: mainContent } = parseFrontmatter(content);
+        const relativePath = path.relative(base, fullPath);
+        files.push({
+          path: relativePath,
+          content: mainContent,
+          title: frontmatter.title || extractTitle(mainContent),
+          description: frontmatter.description,
+          frontmatter
+        });
+      }
+    }
+  } catch (error) {
+    console.warn(`Warning: Could not read directory ${dir}:`, error);
+  }
+  return files;
+}
+function generateLlmsTxt(config) {
+  const lines = [
+    `# ${config.title}`,
+    ""
+  ];
+  if (config.description) {
+    lines.push(`> ${config.description}`);
+    lines.push("");
+  }
+  lines.push("## About");
+  lines.push("");
+  lines.push("This file provides a structured overview of the documentation and content available on this site,");
+  lines.push("optimized for consumption by Large Language Models (LLMs) and AI assistants.");
+  lines.push("");
+  if (config.pages && config.pages.length > 0) {
+    lines.push("## Pages");
+    lines.push("");
+    for (const page of config.pages) {
+      const url = `${config.url}${page.pathname === "/" ? "" : page.pathname}`;
+      const title = page.title || page.pathname;
+      lines.push(`- [${title}](${url})`);
+      if (page.description) {
+        lines.push(`  ${page.description}`);
+      }
+    }
+    lines.push("");
+  }
+  const markdownFiles = collectMarkdownFiles(config.contentDir);
+  if (markdownFiles.length > 0) {
+    lines.push("## Documentation");
+    lines.push("");
+    const grouped = {};
+    for (const file of markdownFiles) {
+      const dir = file.path.split("/")[0] || "root";
+      if (!grouped[dir]) grouped[dir] = [];
+      grouped[dir].push(file);
+    }
+    for (const [dir, files] of Object.entries(grouped)) {
+      lines.push(`### ${dir === "root" ? "Main Documentation" : dir}`);
+      lines.push("");
+      for (const file of files) {
+        const url = `${config.url}/${file.path.replace(/\.mdx?$/, "")}`;
+        lines.push(`- [${file.title}](${url})`);
+        if (file.description) {
+          lines.push(`  ${file.description}`);
+        }
+      }
+      lines.push("");
+    }
+  }
+  lines.push("## Quick Links");
+  lines.push("");
+  lines.push(`- Full Documentation: ${config.url}/llms-full.txt`);
+  lines.push(`- Documentation Manifest: ${config.url}/docs.json`);
+  lines.push(`- AI-Optimized Index: ${config.url}/ai-index.json`);
+  lines.push(`- Sitemap: ${config.url}/sitemap.xml`);
+  lines.push("");
+  lines.push("## For LLMs");
+  lines.push("");
+  lines.push("To get the complete documentation in a single file, request:");
+  lines.push(`${config.url}/llms-full.txt`);
+  lines.push("");
+  lines.push("For structured access to individual pages with metadata:");
+  lines.push(`${config.url}/docs.json`);
+  lines.push("");
+  lines.push("For RAG (Retrieval Augmented Generation) systems:");
+  lines.push(`${config.url}/ai-index.json`);
+  lines.push("");
+  lines.push("---");
+  lines.push("Generated by aeo.js - Answer Engine Optimization for the modern web");
+  lines.push("Learn more at https://aeojs.org");
+  return lines.join("\n");
+}
+function collectAndConcatenateMarkdown(dir, base = dir) {
+  const sections = [];
+  try {
+    const entries = fs.readdirSync(dir).sort();
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry);
+      const stat = fs.statSync(fullPath);
+      if (stat.isDirectory() && !entry.startsWith(".") && entry !== "node_modules") {
+        const subSections = collectAndConcatenateMarkdown(fullPath, base);
+        if (subSections.length > 0) {
+          sections.push(...subSections);
+        }
+      } else if (stat.isFile() && (path.extname(entry) === ".md" || path.extname(entry) === ".mdx")) {
+        const content = fs.readFileSync(fullPath, "utf-8");
+        const { frontmatter, content: mainContent } = parseFrontmatter(content);
+        const relativePath = path.relative(base, fullPath);
+        const sectionLines = [
+          "---",
+          "",
+          `# ${frontmatter.title || relativePath}`,
+          "",
+          `Source: ${relativePath}`,
+          ""
+        ];
+        if (frontmatter.description) {
+          sectionLines.push(`> ${frontmatter.description}`);
+          sectionLines.push("");
+        }
+        const bumpedContent = bumpHeadings(mainContent, 1);
+        sectionLines.push(bumpedContent);
+        sectionLines.push("");
+        sections.push(sectionLines.join("\n"));
+      }
+    }
+  } catch (error) {
+    console.warn(`Warning: Could not read directory ${dir}:`, error);
+  }
+  return sections;
+}
+function generateLlmsFullTxt(config) {
+  const lines = [
+    `# ${config.title} - Complete Documentation`,
+    "",
+    `This file contains all documentation concatenated into a single file for easy consumption by LLMs.`,
+    ""
+  ];
+  if (config.description) {
+    lines.push(`> ${config.description}`);
+    lines.push("");
+  }
+  lines.push("## Table of Contents");
+  lines.push("");
+  lines.push("This document includes all content from this project.");
+  lines.push("Each section is separated by a horizontal rule (---) for easy parsing.");
+  lines.push("");
+  let hasContent = false;
+  if (config.pages && config.pages.length > 0) {
+    for (const page of config.pages) {
+      const url = `${config.url}${page.pathname === "/" ? "" : page.pathname}`;
+      const title = page.title || page.pathname;
+      const sectionLines = [
+        "---",
+        "",
+        `# ${title}`,
+        "",
+        `URL: ${url}`,
+        ""
+      ];
+      if (page.description) {
+        sectionLines.push(`> ${page.description}`);
+        sectionLines.push("");
+      }
+      if (page.content) {
+        sectionLines.push(page.content);
+        sectionLines.push("");
+      }
+      lines.push(sectionLines.join("\n"));
+      hasContent = true;
+    }
+  }
+  const sections = collectAndConcatenateMarkdown(config.contentDir);
+  if (sections.length > 0) {
+    lines.push(...sections);
+    hasContent = true;
+  }
+  if (!hasContent) {
+    lines.push("---");
+    lines.push("");
+    lines.push(`# ${config.title}`);
+    lines.push("");
+    lines.push(`URL: ${config.url}`);
+    lines.push("");
+    if (config.description) {
+      lines.push(config.description);
+      lines.push("");
+    }
+  }
+  lines.push("---");
+  lines.push("");
+  lines.push("## About This Document");
+  lines.push("");
+  lines.push("This concatenated documentation file is generated automatically by aeo.js");
+  lines.push("to make it easier for AI systems to understand the complete context of this project.");
+  lines.push("");
+  lines.push(`For a structured index, see: ${config.url}/llms.txt`);
+  lines.push(`For individual files, see: ${config.url}/docs.json`);
+  lines.push("");
+  lines.push("Generated by aeo.js - https://aeojs.org");
+  return lines.join("\n");
+}
+function ensureDir(path) {
+  fs.mkdirSync(path, { recursive: true });
+}
+function copyMarkdownFiles(config) {
+  const copiedFiles = [];
+  function copyRecursive(dir, base = config.contentDir) {
+    try {
+      const entries = fs.readdirSync(dir);
+      for (const entry of entries) {
+        const fullPath = path.join(dir, entry);
+        const stat = fs.statSync(fullPath);
+        if (stat.isDirectory() && !entry.startsWith(".") && entry !== "node_modules") {
+          copyRecursive(fullPath, base);
+        } else if (stat.isFile() && path.extname(entry) === ".md") {
+          const relativePath = path.relative(base, fullPath);
+          const destPath = path.join(config.outDir, relativePath);
+          ensureDir(path.dirname(destPath));
+          try {
+            fs.copyFileSync(fullPath, destPath);
+            copiedFiles.push({
+              source: fullPath,
+              destination: destPath
+            });
+          } catch (error) {
+            console.warn(`Warning: Could not copy ${fullPath}:`, error);
+          }
+        }
+      }
+    } catch (error) {
+      console.warn(`Warning: Could not read directory ${dir}:`, error);
+    }
+  }
+  copyRecursive(config.contentDir);
+  return copiedFiles;
+}
+function generatePageMarkdownFiles(config) {
+  const generated = [];
+  const pages = config.pages || [];
+  for (const page of pages) {
+    const pageTitle = page.title || (page.pathname === "/" ? config.title : void 0);
+    if (!page.content && !pageTitle) continue;
+    let filename;
+    if (page.pathname === "/") {
+      filename = "index.md";
+    } else {
+      const clean = page.pathname.replace(/^\//, "").replace(/\/$/, "");
+      filename = clean.includes("/") ? `${clean}.md` : `${clean}.md`;
+    }
+    const destPath = path.join(config.outDir, filename);
+    const pageUrl = page.pathname === "/" ? config.url : `${config.url.replace(/\/$/, "")}${page.pathname}`;
+    const lines = [];
+    lines.push("---");
+    if (pageTitle) lines.push(`title: "${pageTitle}"`);
+    if (page.description) lines.push(`description: "${page.description}"`);
+    lines.push(`url: ${pageUrl}`);
+    lines.push(`source: ${pageUrl}`);
+    lines.push(`generated_by: aeo.js`);
+    lines.push("---", "");
+    if (pageTitle) {
+      lines.push(`# ${pageTitle}`, "");
+    }
+    if (page.description) {
+      lines.push(`${page.description}`, "");
+    }
+    if (page.content) {
+      lines.push(page.content);
+    }
+    const content = lines.join("\n");
+    ensureDir(path.dirname(destPath));
+    try {
+      fs.writeFileSync(destPath, content, "utf-8");
+      generated.push({ pathname: page.pathname, destination: destPath });
+    } catch {
+    }
+  }
+  return generated;
+}
+function collectManifestEntries(dir, config, base = dir) {
+  const entries = [];
+  try {
+    const files = fs.readdirSync(dir);
+    for (const file of files) {
+      const fullPath = path.join(dir, file);
+      const stat = fs.statSync(fullPath);
+      if (stat.isDirectory() && !file.startsWith(".") && file !== "node_modules") {
+        entries.push(...collectManifestEntries(fullPath, config, base));
+      } else if (stat.isFile() && (path.extname(file) === ".md" || path.extname(file) === ".mdx")) {
+        const content = fs.readFileSync(fullPath, "utf-8");
+        const { frontmatter, content: mainContent } = parseFrontmatter(content);
+        const relativePath = path.relative(base, fullPath);
+        const urlPath = relativePath.replace(/\.mdx?$/, "");
+        entries.push({
+          url: `${config.url}/${urlPath}`,
+          title: frontmatter.title || extractTitle(mainContent),
+          description: frontmatter.description,
+          lastModified: stat.mtime.toISOString()
+        });
+      }
+    }
+  } catch (error) {
+    console.warn(`Warning: Could not read directory ${dir}:`, error);
+  }
+  return entries;
+}
+function generateManifest(config) {
+  const entries = [];
+  if (config.pages && config.pages.length > 0) {
+    for (const page of config.pages) {
+      entries.push({
+        url: `${config.url}${page.pathname === "/" ? "" : page.pathname}`,
+        title: page.title || page.pathname,
+        description: page.description
+      });
+    }
+  }
+  entries.push(...collectManifestEntries(config.contentDir, config));
+  const manifest = {
+    version: "1.0",
+    generated: (/* @__PURE__ */ new Date()).toISOString(),
+    site: {
+      title: config.title,
+      description: config.description,
+      url: config.url
+    },
+    documents: entries.sort((a, b) => a.url.localeCompare(b.url)),
+    metadata: {
+      totalDocuments: entries.length,
+      generator: "aeo.js",
+      generatorUrl: "https://aeojs.org"
+    }
+  };
+  return JSON.stringify(manifest, null, 2);
+}
+function collectUrls(dir, config, base = dir) {
+  const urls = [];
+  try {
+    const entries = fs.readdirSync(dir);
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry);
+      const stat = fs.statSync(fullPath);
+      if (stat.isDirectory() && !entry.startsWith(".") && entry !== "node_modules") {
+        urls.push(...collectUrls(fullPath, config, base));
+      } else if (stat.isFile() && (path.extname(entry) === ".md" || path.extname(entry) === ".mdx" || path.extname(entry) === ".html")) {
+        const relativePath = path.relative(base, fullPath);
+        const urlPath = relativePath.replace(/\.(md|mdx|html)$/, "");
+        urls.push(`${config.url}/${urlPath}`);
+      }
+    }
+  } catch (error) {
+    console.warn(`Warning: Could not read directory ${dir}:`, error);
+  }
+  return urls;
+}
+function escapeXml(str) {
+  return str.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+function generateSitemap(config) {
+  const urls = [];
+  if (config.pages && config.pages.length > 0) {
+    for (const page of config.pages) {
+      urls.push(`${config.url}${page.pathname === "/" ? "" : page.pathname}`);
+    }
+  }
+  if (config.contentDir) {
+    urls.push(...collectUrls(config.contentDir, config));
+  }
+  const lines = [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+  ];
+  urls.push(config.url);
+  const uniqueUrls = [...new Set(urls)].sort();
+  for (const url of uniqueUrls) {
+    lines.push("  <url>");
+    lines.push(`    <loc>${escapeXml(url)}</loc>`);
+    lines.push(`    <lastmod>${(/* @__PURE__ */ new Date()).toISOString().split("T")[0]}</lastmod>`);
+    lines.push("    <changefreq>weekly</changefreq>");
+    lines.push("    <priority>0.8</priority>");
+    lines.push("  </url>");
+  }
+  lines.push("</urlset>");
+  return lines.join("\n");
+}
+function extractKeywords(content) {
+  const words = content.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter((word) => word.length > 3);
+  const wordCount = {};
+  for (const word of words) {
+    wordCount[word] = (wordCount[word] || 0) + 1;
+  }
+  return Object.entries(wordCount).sort((a, b) => b[1] - a[1]).slice(0, 10).map(([word]) => word);
+}
+function chunkContent(content, maxLength = 2e3) {
+  const chunks = [];
+  const paragraphs = content.split("\n\n");
+  let currentChunk = "";
+  for (const paragraph of paragraphs) {
+    if (currentChunk.length + paragraph.length > maxLength && currentChunk.length > 0) {
+      chunks.push(currentChunk.trim());
+      currentChunk = "";
+    }
+    currentChunk += paragraph + "\n\n";
+  }
+  if (currentChunk.trim()) {
+    chunks.push(currentChunk.trim());
+  }
+  return chunks;
+}
+function collectAIIndexEntries(dir, config, base = dir) {
+  const entries = [];
+  try {
+    const files = fs.readdirSync(dir);
+    for (const file of files) {
+      const fullPath = path.join(dir, file);
+      const stat = fs.statSync(fullPath);
+      if (stat.isDirectory() && !file.startsWith(".") && file !== "node_modules") {
+        entries.push(...collectAIIndexEntries(fullPath, config, base));
+      } else if (stat.isFile() && (path.extname(file) === ".md" || path.extname(file) === ".mdx")) {
+        const content = fs.readFileSync(fullPath, "utf-8");
+        const { frontmatter, content: mainContent } = parseFrontmatter(content);
+        const relativePath = path.relative(base, fullPath);
+        const urlPath = relativePath.replace(/\.mdx?$/, "");
+        const url = `${config.url}/${urlPath}`;
+        const chunks = chunkContent(mainContent);
+        const title = frontmatter.title || extractTitle(mainContent);
+        const keywords = extractKeywords(mainContent);
+        chunks.forEach((chunk, index) => {
+          const id = crypto.createHash("sha256").update(`${url}-${index}`).digest("hex").slice(0, 16);
+          entries.push({
+            id,
+            url,
+            title: chunks.length > 1 ? `${title} (Part ${index + 1})` : title,
+            content: chunk,
+            description: frontmatter.description,
+            keywords,
+            metadata: {
+              ...frontmatter,
+              chunkIndex: index,
+              totalChunks: chunks.length,
+              sourcePath: relativePath
+            }
+          });
+        });
+      }
+    }
+  } catch (error) {
+    console.warn(`Warning: Could not read directory ${dir}:`, error);
+  }
+  return entries;
+}
+function generateAIIndex(config) {
+  const entries = [];
+  if (config.pages && config.pages.length > 0) {
+    for (const page of config.pages) {
+      const url = `${config.url}${page.pathname === "/" ? "" : page.pathname}`;
+      const title = page.title || page.pathname;
+      const content = page.content || "";
+      if (content) {
+        const chunks = chunkContent(content);
+        const keywords = extractKeywords(content);
+        chunks.forEach((chunk, index2) => {
+          const id = crypto.createHash("sha256").update(`${url}-${index2}`).digest("hex").slice(0, 16);
+          entries.push({
+            id,
+            url,
+            title: chunks.length > 1 ? `${title} (Part ${index2 + 1})` : title,
+            content: chunk,
+            description: page.description,
+            keywords,
+            metadata: {
+              chunkIndex: index2,
+              totalChunks: chunks.length,
+              sourcePath: page.pathname
+            }
+          });
+        });
+      } else {
+        const id = crypto.createHash("sha256").update(url).digest("hex").slice(0, 16);
+        entries.push({
+          id,
+          url,
+          title,
+          content: page.description || title,
+          description: page.description,
+          keywords: []
+        });
+      }
+    }
+  }
+  entries.push(...collectAIIndexEntries(config.contentDir, config));
+  const index = {
+    version: "1.0",
+    generated: (/* @__PURE__ */ new Date()).toISOString(),
+    site: {
+      title: config.title,
+      description: config.description,
+      url: config.url
+    },
+    entries: entries.sort((a, b) => a.id.localeCompare(b.id)),
+    metadata: {
+      totalEntries: entries.length,
+      generator: "aeo.js",
+      generatorUrl: "https://aeojs.org",
+      embedding: {
+        recommended: "text-embedding-ada-002",
+        dimensions: 1536
+      }
+    }
+  };
+  return JSON.stringify(index, null, 2);
+}
+async function generateAEOFiles(configOrRoot, maybeConfig) {
+  var _a;
+  let config;
+  if (typeof configOrRoot === "string") {
+    config = resolveConfig({ ...maybeConfig, outDir: configOrRoot });
+  } else if (configOrRoot && typeof configOrRoot === "object" && "generators" in configOrRoot && typeof ((_a = configOrRoot.generators) == null ? void 0 : _a.robotsTxt) === "boolean") {
+    config = configOrRoot;
+  } else {
+    config = resolveConfig(configOrRoot);
+  }
+  const outDir = config.outDir;
+  const files = [];
+  const errors = [];
+  if (!fs.existsSync(outDir)) {
+    fs.mkdirSync(outDir, { recursive: true });
+  }
+  if (config.generators.robotsTxt) {
+    try {
+      const content = generateRobotsTxt(config);
+      fs.writeFileSync(path.join(outDir, "robots.txt"), content, "utf-8");
+      files.push("robots.txt");
+    } catch (e) {
+      errors.push(`robots.txt: ${e.message}`);
+    }
+  }
+  if (config.generators.llmsTxt) {
+    try {
+      const content = generateLlmsTxt(config);
+      fs.writeFileSync(path.join(outDir, "llms.txt"), "\uFEFF" + content, "utf-8");
+      files.push("llms.txt");
+    } catch (e) {
+      errors.push(`llms.txt: ${e.message}`);
+    }
+  }
+  if (config.generators.llmsFullTxt) {
+    try {
+      const content = generateLlmsFullTxt(config);
+      fs.writeFileSync(path.join(outDir, "llms-full.txt"), "\uFEFF" + content, "utf-8");
+      files.push("llms-full.txt");
+    } catch (e) {
+      errors.push(`llms-full.txt: ${e.message}`);
+    }
+  }
+  if (config.generators.rawMarkdown) {
+    try {
+      const generated = generatePageMarkdownFiles(config);
+      for (const f of generated) {
+        files.push(f.destination);
+      }
+    } catch (e) {
+      errors.push(`page-markdown: ${e.message}`);
+    }
+    try {
+      const copied = copyMarkdownFiles(config);
+      for (const f of copied) {
+        files.push(f.destination);
+      }
+    } catch (e) {
+      errors.push(`raw-markdown: ${e.message}`);
+    }
+  }
+  if (config.generators.manifest) {
+    try {
+      const content = generateManifest(config);
+      fs.writeFileSync(path.join(outDir, "docs.json"), content, "utf-8");
+      files.push("docs.json");
+    } catch (e) {
+      errors.push(`docs.json: ${e.message}`);
+    }
+  }
+  if (config.generators.sitemap) {
+    try {
+      const content = generateSitemap(config);
+      fs.writeFileSync(path.join(outDir, "sitemap.xml"), content, "utf-8");
+      files.push("sitemap.xml");
+    } catch (e) {
+      errors.push(`sitemap.xml: ${e.message}`);
+    }
+  }
+  if (config.generators.aiIndex) {
+    try {
+      const content = generateAIIndex(config);
+      fs.writeFileSync(path.join(outDir, "ai-index.json"), content, "utf-8");
+      files.push("ai-index.json");
+    } catch (e) {
+      errors.push(`ai-index.json: ${e.message}`);
+    }
+  }
+  return { files, errors };
+}
+function scanBuiltPages(dir, baseUrl) {
+  const pages = [];
+  function walk(currentDir) {
+    var _a;
+    try {
+      const entries = fs.readdirSync(currentDir);
+      for (const entry of entries) {
+        const fullPath = path.join(currentDir, entry);
+        const stat = fs.statSync(fullPath);
+        if (stat.isDirectory() && !entry.startsWith(".") && entry !== "_astro") {
+          walk(fullPath);
+        } else if (entry === "index.html" || entry.endsWith(".html") && entry !== "404.html" && entry !== "500.html") {
+          try {
+            const html = fs.readFileSync(fullPath, "utf-8");
+            const titleMatch = html.match(/<title>([^<]*)<\/title>/i);
+            const descMatch = html.match(/<meta\s+name=["']description["']\s+content=["']([^"']*)["']/i);
+            const textContent = extractTextFromHtml(html);
+            let pathname;
+            const relative8 = fullPath.slice(dir.length);
+            if (entry === "index.html") {
+              pathname = "/" + relative8.replace(/\/?index\.html$/, "");
+              if (pathname !== "/") pathname = pathname.replace(/\/$/, "");
+            } else {
+              pathname = "/" + relative8.replace(/\.html$/, "");
+            }
+            pathname = pathname.replace(/\/+/g, "/") || "/";
+            const rawTitle = titleMatch ? titleMatch[1] : void 0;
+            const title = ((_a = rawTitle == null ? void 0 : rawTitle.split("|")[0]) == null ? void 0 : _a.trim()) || rawTitle;
+            pages.push({
+              pathname,
+              title,
+              description: descMatch ? descMatch[1] : void 0,
+              content: textContent
+            });
+          } catch {
+          }
+        }
+      }
+    } catch {
+    }
+  }
+  walk(dir);
+  return pages;
+}
+function scanDevPages(pagesDir) {
+  const pages = [];
+  function walk(currentDir, base) {
+    try {
+      const entries = fs.readdirSync(currentDir);
+      for (const entry of entries) {
+        const fullPath = path.join(currentDir, entry);
+        const stat = fs.statSync(fullPath);
+        if (stat.isDirectory() && !entry.startsWith(".") && !entry.startsWith("_")) {
+          walk(fullPath, base);
+        } else if (entry.endsWith(".astro") || entry.endsWith(".md") || entry.endsWith(".mdx")) {
+          if (entry.startsWith("404") || entry.startsWith("500") || entry.startsWith("[")) continue;
+          const relative8 = fullPath.slice(base.length);
+          let pathname = "/" + relative8.replace(/\.(astro|md|mdx)$/, "");
+          if (pathname.endsWith("/index")) pathname = pathname.slice(0, -6) || "/";
+          pathname = pathname.replace(/\/+/g, "/") || "/";
+          const name = entry.replace(/\.(astro|md|mdx)$/, "");
+          pages.push({
+            pathname,
+            title: name === "index" ? void 0 : name.charAt(0).toUpperCase() + name.slice(1)
+          });
+        }
+      }
+    } catch {
+    }
+  }
+  const resolvedPagesDir = path.join(process.cwd(), pagesDir);
+  if (fs.existsSync(resolvedPagesDir)) {
+    walk(resolvedPagesDir, resolvedPagesDir);
+  }
+  return pages;
+}
+function extractTextFromHtml(html) {
+  let text = html;
+  text = text.replace(/<script[\s\S]*?<\/script>/gi, "");
+  text = text.replace(/<style[\s\S]*?<\/style>/gi, "");
+  text = text.replace(/<svg[\s\S]*?<\/svg>/gi, "");
+  const mainMatch = text.match(/<main[^>]*>([\s\S]*)<\/main>/i);
+  if (mainMatch) {
+    text = mainMatch[1];
+  } else {
+    text = text.replace(/<nav[\s\S]*?<\/nav>/gi, "");
+    text = text.replace(/<header[\s\S]*?<\/header>/gi, "");
+    text = text.replace(/<footer[\s\S]*?<\/footer>/gi, "");
+  }
+  text = text.replace(/<a[^>]+href=["']([^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi, (_, url, inner) => {
+    if (/<(?:h[1-6]|div|p|section)[^>]*>/i.test(inner)) {
+      const cleanInner = inner.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+      return `
+[${cleanInner.slice(0, 120).trim()}](${url})
+`;
+    }
+    return `[${inner}](${url})`;
+  });
+  text = text.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, "\n\n## $1\n\n");
+  text = text.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, "\n\n## $1\n\n");
+  text = text.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, "\n\n### $1\n\n");
+  text = text.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, "\n\n#### $1\n\n");
+  text = text.replace(/<h5[^>]*>([\s\S]*?)<\/h5>/gi, "\n\n##### $1\n\n");
+  text = text.replace(/<h6[^>]*>([\s\S]*?)<\/h6>/gi, "\n\n###### $1\n\n");
+  text = text.replace(/<a[^>]+href=["']([^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi, "[$2]($1)");
+  text = text.replace(/<(?:strong|b)[^>]*>([\s\S]*?)<\/(?:strong|b)>/gi, "**$1**");
+  text = text.replace(/<(?:em|i)[^>]*>([\s\S]*?)<\/(?:em|i)>/gi, "*$1*");
+  text = text.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, "\n- $1");
+  text = text.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, "\n\n> $1\n\n");
+  text = text.replace(/<hr[^>]*\/?>/gi, "\n\n---\n\n");
+  text = text.replace(/<br[^>]*\/?>/gi, "\n");
+  text = text.replace(/<\/p>/gi, "\n\n");
+  text = text.replace(/<p[^>]*>/gi, "");
+  text = text.replace(/<\/?(?:div|section|article|header|main|aside|figure|figcaption|table|thead|tbody|tr|td|th|ul|ol|dl|dt|dd)[^>]*>/gi, "\n");
+  text = text.replace(/<[^>]+>/g, "");
+  text = text.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, " ").replace(/&copy;/g, "(c)");
+  text = text.replace(/[\u{1F1E0}-\u{1FAFF}\u{2600}-\u{27BF}\u{FE00}-\u{FE0F}\u{200D}\u{20E3}]/gu, "");
+  text = text.split("\n").map((l) => l.replace(/\s+/g, " ").trim()).join("\n");
+  text = text.replace(/\n{3,}/g, "\n\n");
+  text = text.replace(/\[[\s\n]+/g, "[").replace(/[\s\n]+\]/g, "]");
+  text = text.replace(/(#{2,6})\s*\n+\s*/g, "$1 ");
+  text = text.replace(/^#{2,6}\s*$/gm, "");
+  text = text.replace(/\n{3,}/g, "\n\n");
+  return text.trim().slice(0, 8e3);
+}
+function htmlToMarkdown(html, pagePath, config) {
+  var _a, _b;
+  const titleMatch = html.match(/<title>([^<]*)<\/title>/i);
+  const descMatch = html.match(/<meta\s+name=["']description["']\s+content=["']([^"']*)["']/i);
+  const textContent = extractTextFromHtml(html);
+  const rawTitle = titleMatch ? (_b = (_a = titleMatch[1]) == null ? void 0 : _a.split("|")[0]) == null ? void 0 : _b.trim() : void 0;
+  const description = descMatch == null ? void 0 : descMatch[1];
+  const pageUrl = pagePath === "/" ? config.url : `${config.url.replace(/\/$/, "")}${pagePath}`;
+  const lines = [];
+  lines.push("---");
+  if (rawTitle) lines.push(`title: "${rawTitle}"`);
+  if (description) lines.push(`description: "${description}"`);
+  lines.push(`url: ${pageUrl}`);
+  lines.push(`source: ${pageUrl}`);
+  lines.push(`generated_by: aeo.js`);
+  lines.push("---", "");
+  if (rawTitle) lines.push(`# ${rawTitle}`, "");
+  if (description) lines.push(`${description}`, "");
+  if (textContent) lines.push(textContent);
+  return lines.join("\n");
+}
+function aeoAstroIntegration(options = {}) {
+  let resolvedConfig = resolveConfig(options);
+  let astroConfig;
+  return {
+    name: "aeo-astro",
+    hooks: {
+      "astro:config:setup": ({ config, command, injectScript }) => {
+        astroConfig = config;
+        resolvedConfig = resolveConfig({
+          ...options,
+          contentDir: options.contentDir || "src/content",
+          outDir: options.outDir || (command === "build" ? config.outDir.pathname : config.publicDir.pathname)
+        });
+        if (command === "dev") {
+          const publicPath = config.publicDir.pathname;
+          if (!fs.existsSync(publicPath)) {
+            fs.mkdirSync(publicPath, { recursive: true });
+          }
+        }
+        if (resolvedConfig.widget.enabled && injectScript) {
+          const widgetOpts = JSON.stringify(resolvedConfig.widget);
+          injectScript(
+            "page",
+            `import { AeoWidget } from 'aeo.js/widget';
+if (document.readyState === 'loading') {
+  document.addEventListener('DOMContentLoaded', () => { new AeoWidget(${widgetOpts}); });
+} else {
+  new AeoWidget(${widgetOpts});
+}`
+          );
+        }
+      },
+      "astro:build:done": async ({ dir, logger }) => {
+        const buildLogger = logger.fork("aeo.js");
+        buildLogger.info("Generating AEO files...");
+        const outPath = dir instanceof URL ? dir.pathname : dir || astroConfig.outDir.pathname;
+        options.url || astroConfig.site || "https://example.com";
+        const discoveredPages = scanBuiltPages(outPath);
+        buildLogger.info(`Discovered ${discoveredPages.length} pages from build output`);
+        resolvedConfig = resolveConfig({
+          ...options,
+          outDir: options.outDir || outPath,
+          pages: [...options.pages || [], ...discoveredPages]
+        });
+        try {
+          const result = await generateAEOFiles(resolvedConfig);
+          if (result.files.length > 0) {
+            buildLogger.info(`Generated ${result.files.length} files`);
+            result.files.forEach((file) => {
+              buildLogger.debug(`  - ${file}`);
+            });
+          }
+          if (result.errors.length > 0) {
+            buildLogger.error("Errors during generation:");
+            result.errors.forEach((error) => {
+              buildLogger.error(`  - ${error}`);
+            });
+          }
+        } catch (error) {
+          buildLogger.error(`Failed to generate AEO files: ${error}`);
+        }
+      },
+      "astro:server:setup": async ({ server, logger }) => {
+        const devLogger = logger.fork("aeo.js");
+        devLogger.info("Generating AEO files for development...");
+        const devPages = scanDevPages("src/pages");
+        resolvedConfig = resolveConfig({
+          ...options,
+          contentDir: options.contentDir || "src/content",
+          outDir: resolvedConfig.outDir,
+          pages: [...options.pages || [], ...devPages]
+        });
+        try {
+          const result = await generateAEOFiles(resolvedConfig);
+          if (result.files.length > 0) {
+            devLogger.info(`Generated ${result.files.length} files`);
+          }
+          if (result.errors.length > 0) {
+            devLogger.error("Errors during generation:", result.errors);
+          }
+        } catch (error) {
+          devLogger.error(`Failed to generate AEO files: ${error}`);
+        }
+        const mdHandler = async (req, res, next) => {
+          var _a, _b;
+          if (!((_a = req.url) == null ? void 0 : _a.endsWith(".md"))) return next();
+          if (req.headers["x-aeo-internal"]) return next();
+          const filename = req.url.startsWith("/") ? req.url.slice(1) : req.url;
+          if (resolvedConfig.contentDir) {
+            const contentFile = path.join(process.cwd(), resolvedConfig.contentDir, filename);
+            if (fs.existsSync(contentFile)) {
+              res.setHeader("Content-Type", "text/markdown; charset=utf-8");
+              res.end(fs.readFileSync(contentFile, "utf-8"));
+              return;
+            }
+          }
+          let pagePath = req.url.replace(/\.md$/, "") || "/";
+          if (pagePath === "/index") pagePath = "/";
+          try {
+            const host = req.headers.host || "localhost:4321";
+            const protocol = ((_b = req.connection) == null ? void 0 : _b.encrypted) ? "https" : "http";
+            const response = await fetch(`${protocol}://${host}${pagePath}`, {
+              headers: { "x-aeo-internal": "1" }
+            });
+            if (response.ok) {
+              const html = await response.text();
+              const md = htmlToMarkdown(html, pagePath, resolvedConfig);
+              res.setHeader("Content-Type", "text/markdown; charset=utf-8");
+              res.end(md);
+              return;
+            }
+          } catch {
+          }
+          const filepath = path.join(resolvedConfig.outDir, filename);
+          if (fs.existsSync(filepath)) {
+            res.setHeader("Content-Type", "text/markdown; charset=utf-8");
+            res.end(fs.readFileSync(filepath, "utf-8"));
+            return;
+          }
+          next();
+        };
+        server.middlewares.stack.unshift({ route: "", handle: mdHandler });
+        if (resolvedConfig.contentDir) {
+          const contentPath = path.join(process.cwd(), resolvedConfig.contentDir);
+          server.watcher.add(path.join(contentPath, "**/*.md"));
+          server.watcher.add(path.join(contentPath, "**/*.mdx"));
+          server.watcher.on("change", async (file) => {
+            if (file.endsWith(".md") || file.endsWith(".mdx")) {
+              devLogger.info("Content file changed, regenerating AEO files...");
+              try {
+                const result = await generateAEOFiles(resolvedConfig);
+                if (result.files.length > 0) {
+                  devLogger.info(`Regenerated ${result.files.length} files`);
+                }
+                if (result.errors.length > 0) {
+                  devLogger.error("Errors during regeneration:", result.errors);
+                }
+              } catch (error) {
+                devLogger.error(`Failed to regenerate AEO files: ${error}`);
+              }
+            }
+          });
+        }
+      }
+    }
+  };
+}
+var AeoMetaTags = ({ config }) => {
+  const resolvedConfig = resolveConfig(config);
+  return `
+    <link rel="alternate" type="text/plain" href="/llms.txt" title="LLM Summary" />
+    <link rel="alternate" type="text/plain" href="/llms-full.txt" title="Full Content for LLMs" />
+    <link rel="alternate" type="application/json" href="/docs.json" title="Documentation Manifest" />
+    <link rel="alternate" type="application/json" href="/ai-index.json" title="AI-Optimized Index" />
+    <meta name="aeo:title" content="${resolvedConfig.title}" />
+    <meta name="aeo:description" content="${resolvedConfig.description}" />
+    <meta name="aeo:url" content="${resolvedConfig.url}" />
+  `;
+};
+function defineAeoConfig(config) {
+  return config;
+}
+var astro_default = aeoAstroIntegration;
+
+exports.AeoMetaTags = AeoMetaTags;
+exports.aeoAstroIntegration = aeoAstroIntegration;
+exports.default = astro_default;
+exports.defineAeoConfig = defineAeoConfig;
+//# sourceMappingURL=astro.js.map
+//# sourceMappingURL=astro.js.map