@agent-seo/core 1.0.0

package/dist/index.js

@@ -0,0 +1,867 @@
+// src/detect.ts
+var AI_BOT_REGISTRY = [
+  // === OpenAI ===
+  {
+    pattern: /GPTBot/i,
+    info: { name: "GPTBot", operator: "OpenAI", purpose: "training", rendersJs: false }
+  },
+  {
+    pattern: /OAI-SearchBot/i,
+    info: { name: "OAI-SearchBot", operator: "OpenAI", purpose: "search", rendersJs: false }
+  },
+  {
+    pattern: /ChatGPT-User/i,
+    info: { name: "ChatGPT-User", operator: "OpenAI", purpose: "agent-browsing", rendersJs: true }
+  },
+  // === Anthropic ===
+  {
+    pattern: /ClaudeBot/i,
+    info: { name: "ClaudeBot", operator: "Anthropic", purpose: "training", rendersJs: false }
+  },
+  {
+    pattern: /Claude-User/i,
+    info: { name: "Claude-User", operator: "Anthropic", purpose: "agent-browsing", rendersJs: true }
+  },
+  {
+    pattern: /Claude-SearchBot/i,
+    info: { name: "Claude-SearchBot", operator: "Anthropic", purpose: "search", rendersJs: false }
+  },
+  {
+    pattern: /anthropic-ai/i,
+    info: { name: "anthropic-ai", operator: "Anthropic", purpose: "training", rendersJs: false }
+  },
+  // === Perplexity ===
+  {
+    pattern: /PerplexityBot/i,
+    info: { name: "PerplexityBot", operator: "Perplexity", purpose: "search", rendersJs: false }
+  },
+  {
+    pattern: /Perplexity-User/i,
+    info: { name: "Perplexity-User", operator: "Perplexity", purpose: "agent-browsing", rendersJs: true }
+  },
+  // === Google ===
+  {
+    pattern: /Google-Extended/i,
+    info: { name: "Google-Extended", operator: "Google", purpose: "training", rendersJs: true }
+  },
+  // === Apple ===
+  {
+    pattern: /Applebot-Extended/i,
+    info: { name: "Applebot-Extended", operator: "Apple", purpose: "training", rendersJs: true }
+  },
+  // === Meta ===
+  {
+    pattern: /meta-externalagent/i,
+    info: { name: "Meta-ExternalAgent", operator: "Meta", purpose: "training", rendersJs: false }
+  },
+  {
+    pattern: /FacebookBot/i,
+    info: { name: "FacebookBot", operator: "Meta", purpose: "search", rendersJs: false }
+  },
+  // === Common Crawl ===
+  {
+    pattern: /CCBot/i,
+    info: { name: "CCBot", operator: "Common Crawl", purpose: "training", rendersJs: false }
+  },
+  // === Cohere ===
+  {
+    pattern: /cohere-ai/i,
+    info: { name: "cohere-ai", operator: "Cohere", purpose: "training", rendersJs: false }
+  },
+  // === Amazon ===
+  {
+    pattern: /Amazonbot/i,
+    info: { name: "Amazonbot", operator: "Amazon", purpose: "search", rendersJs: false }
+  },
+  // === Bytedance ===
+  {
+    pattern: /Bytespider/i,
+    info: { name: "Bytespider", operator: "ByteDance", purpose: "training", rendersJs: false }
+  },
+  // === You.com ===
+  {
+    pattern: /YouBot/i,
+    info: { name: "YouBot", operator: "You.com", purpose: "search", rendersJs: false }
+  },
+  // === DeepSeek ===
+  {
+    pattern: /Deepseek/i,
+    info: { name: "DeepSeekBot", operator: "DeepSeek", purpose: "training", rendersJs: false }
+  }
+];
+var TOKEN_REGISTRY = AI_BOT_REGISTRY.map((entry) => ({
+  entry,
+  token: regexToToken(entry.pattern)
+}));
+function detectAgent(userAgent, acceptHeader) {
+  const wantsMarkdown = acceptHeader ? /text\/markdown/i.test(acceptHeader) : false;
+  if (!userAgent) {
+    return { isAIBot: false, bot: null, wantsMarkdown };
+  }
+  const ua = userAgent.toLowerCase();
+  for (const { entry, token } of TOKEN_REGISTRY) {
+    if (token) {
+      if (ua.includes(token)) {
+        return { isAIBot: true, bot: entry.info, wantsMarkdown };
+      }
+      continue;
+    }
+    if (entry.pattern.test(userAgent)) {
+      return { isAIBot: true, bot: entry.info, wantsMarkdown };
+    }
+  }
+  return { isAIBot: false, bot: null, wantsMarkdown };
+}
+function shouldServeMarkdown(userAgent, acceptHeader) {
+  const ctx = detectAgent(userAgent, acceptHeader);
+  return ctx.isAIBot || ctx.wantsMarkdown;
+}
+function regexToToken(pattern) {
+  const source = pattern.source;
+  if (/^[A-Za-z0-9-]+$/.test(source)) return source.toLowerCase();
+  return null;
+}
+
+// src/transform.ts
+import { JSDOM as JSDOM2 } from "jsdom";
+import { Readability, isProbablyReaderable } from "@mozilla/readability";
+
+// src/sanitize.ts
+import { JSDOM } from "jsdom";
+var DEFAULT_STRIP_TAGS = [
+  "script",
+  "style",
+  "noscript",
+  "iframe",
+  "svg",
+  "canvas",
+  "video",
+  "audio",
+  "map",
+  "object",
+  "embed",
+  "applet",
+  'link[rel="stylesheet"]',
+  "meta"
+];
+var DEFAULT_STRIP_SELECTORS = [
+  // Navigation & chrome
+  "nav",
+  "header:not(article header)",
+  "footer:not(article footer)",
+  '[role="navigation"]',
+  '[role="banner"]',
+  '[role="contentinfo"]',
+  '[role="complementary"]',
+  "aside",
+  // Ads, cookies, popups
+  ".advertisement",
+  ".ad",
+  ".ads",
+  '[class*="ad-"]',
+  '[class*="ad_"]',
+  ".cookie-banner",
+  ".cookie-consent",
+  '[class*="cookie"]',
+  ".popup",
+  ".modal",
+  '[class*="popup"]',
+  '[class*="modal"]',
+  ".overlay",
+  // Social & sharing
+  ".social-share",
+  ".share-buttons",
+  '[class*="social"]',
+  '[class*="share"]',
+  ".follow-us",
+  // Comments & forms (not the content)
+  ".comments",
+  "#comments",
+  ".comment-form",
+  'form:not([class*="search"])',
+  // Related content / sidebar noise
+  ".related-posts",
+  ".recommended",
+  ".sidebar",
+  ".widget",
+  '[class*="related"]',
+  '[class*="sidebar"]',
+  '[class*="widget"]',
+  ".newsletter",
+  ".subscribe",
+  '[class*="newsletter"]',
+  ".cta",
+  '[class*="cta"]',
+  // Visual-only elements
+  ".breadcrumb",
+  ".breadcrumbs",
+  ".pagination",
+  ".skip-link",
+  '[aria-hidden="true"]',
+  // JS framework artifacts
+  "[data-reactroot] > noscript",
+  ".hydration-overlay"
+];
+function sanitizeHtml(html, options = {}) {
+  const { stripSelectors = [], preserveSelectors = [] } = options;
+  const dom = new JSDOM(html);
+  const document = dom.window.document;
+  const preserveSet = /* @__PURE__ */ new Set();
+  for (const selector of preserveSelectors) {
+    try {
+      document.querySelectorAll(selector).forEach((el) => preserveSet.add(el));
+    } catch {
+    }
+  }
+  for (const tag of DEFAULT_STRIP_TAGS) {
+    try {
+      document.querySelectorAll(tag).forEach((el) => {
+        if (!preserveSet.has(el)) el.remove();
+      });
+    } catch {
+    }
+  }
+  const allStripSelectors = [...DEFAULT_STRIP_SELECTORS, ...stripSelectors];
+  for (const selector of allStripSelectors) {
+    try {
+      document.querySelectorAll(selector).forEach((el) => {
+        if (!preserveSet.has(el)) el.remove();
+      });
+    } catch {
+    }
+  }
+  stripLowDensityElements(document);
+  removeEmptyElements(document);
+  cleanAttributes(document);
+  const result = document.body?.innerHTML || "";
+  dom.window.close();
+  return result;
+}
+function stripLowDensityElements(document) {
+  const candidates = document.querySelectorAll("div, section, span");
+  for (const el of candidates) {
+    const textLength = (el.textContent || "").trim().length;
+    const childElementCount = el.querySelectorAll("*").length;
+    if (childElementCount > 10 && textLength < 50) {
+      el.remove();
+    }
+  }
+}
+function removeEmptyElements(document) {
+  const candidates = document.querySelectorAll("div, span, p, section, article");
+  for (const el of candidates) {
+    if (!(el.textContent || "").trim() && !el.querySelector("img, table, pre, code")) {
+      el.remove();
+    }
+  }
+}
+function cleanAttributes(document) {
+  const all = document.querySelectorAll("*");
+  const keepAttrs = /* @__PURE__ */ new Set(["href", "src", "alt", "title", "colspan", "rowspan", "scope", "headers", "lang", "dir", "type"]);
+  for (const el of all) {
+    const attrs = Array.from(el.attributes);
+    for (const attr of attrs) {
+      if (attr.name === "class" && el.tagName === "CODE") continue;
+      if (!keepAttrs.has(attr.name)) {
+        el.removeAttribute(attr.name);
+      }
+    }
+  }
+}
+
+// src/markdown.ts
+import TurndownService from "turndown";
+import { gfm } from "turndown-plugin-gfm";
+function htmlToMarkdown(html, options = {}) {
+  const { url, customRules = [] } = options;
+  const turndown = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+    bulletListMarker: "-",
+    emDelimiter: "*",
+    strongDelimiter: "**",
+    linkStyle: "inlined",
+    hr: "---"
+  });
+  turndown.use(gfm);
+  turndown.addRule("fencedCodeBlock", {
+    filter: (node) => {
+      return node.nodeName === "PRE" && node.firstChild !== null && node.firstChild.nodeName === "CODE";
+    },
+    replacement: (_content, node) => {
+      const codeEl = node.firstChild;
+      const className = codeEl?.getAttribute?.("class") || "";
+      const langMatch = className.match(
+        /(?:language-|lang-|hljs\s+|highlight-)([a-zA-Z0-9_+-]+)/
+      );
+      const lang = langMatch ? langMatch[1] : "";
+      const code = codeEl?.textContent || "";
+      return `
+
+\`\`\`${lang}
+${code.replace(/\n+$/, "")}
+\`\`\`
+
+`;
+    }
+  });
+  turndown.addRule("meaningfulImages", {
+    filter: (node) => node.nodeName === "IMG",
+    replacement: (_content, node) => {
+      const el = node;
+      const alt = el.getAttribute("alt")?.trim();
+      const src = el.getAttribute("src")?.trim();
+      if (!alt) return "";
+      let resolvedSrc = src || "";
+      if (url && src && !src.startsWith("http") && !src.startsWith("data:")) {
+        try {
+          resolvedSrc = new URL(src, url).href;
+        } catch {
+          resolvedSrc = src;
+        }
+      }
+      return `![${alt}](${resolvedSrc})`;
+    }
+  });
+  turndown.addRule("resolveLinks", {
+    filter: "a",
+    replacement: (content, node) => {
+      const el = node;
+      const href = el.getAttribute("href");
+      if (!href || !content.trim()) return content;
+      if (href.startsWith("#")) return content;
+      if (href.startsWith("javascript:") || href.startsWith("data:text/html")) return "";
+      let resolvedHref = href;
+      if (url && !href.startsWith("http") && !href.startsWith("mailto:")) {
+        try {
+          resolvedHref = new URL(href, url).href;
+        } catch {
+          resolvedHref = href;
+        }
+      }
+      const title = el.getAttribute("title");
+      return title ? `[${content}](${resolvedHref} "${title}")` : `[${content}](${resolvedHref})`;
+    }
+  });
+  for (const rule of customRules) {
+    turndown.addRule(rule.name, {
+      filter: rule.filter,
+      replacement: rule.replacement
+    });
+  }
+  let markdown = turndown.turndown(html);
+  markdown = markdown.replace(/\n{3,}/g, "\n\n").trim();
+  return markdown;
+}
+
+// src/json-ld.ts
+function extractJsonLdBlocks(document) {
+  const results = [];
+  const scripts = document.querySelectorAll('script[type="application/ld+json"]');
+  for (const script of scripts) {
+    try {
+      const data = JSON.parse(script.textContent || "");
+      if (data["@graph"] && Array.isArray(data["@graph"])) {
+        results.push(...data["@graph"]);
+      } else {
+        results.push(data);
+      }
+    } catch {
+    }
+  }
+  return results;
+}
+
+// src/tokens.ts
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+
+// src/frontmatter.ts
+function buildFrontmatter(input) {
+  const lines = ["---"];
+  if (input.title) lines.push(`title: "${escapeYaml(input.title)}"`);
+  if (input.description)
+    lines.push(`description: "${escapeYaml(input.description)}"`);
+  if (input.url) lines.push(`url: "${escapeYaml(input.url)}"`);
+  if (input.lang) lines.push(`lang: "${escapeYaml(input.lang)}"`);
+  if (input.lastModified) lines.push(`lastModified: "${escapeYaml(input.lastModified)}"`);
+  if (input.jsonLd?.length) {
+    const primary = input.jsonLd[0];
+    const primaryType = primary?.["@type"];
+    if (primaryType) {
+      const typeStr = Array.isArray(primaryType) ? primaryType[0] : primaryType;
+      if (typeof typeStr === "string") {
+        lines.push(`schema: "${escapeYaml(typeStr)}"`);
+      }
+    }
+    const author = primary?.author;
+    if (author) {
+      const authorName = author.name;
+      if (authorName) lines.push(`author: "${escapeYaml(authorName)}"`);
+    }
+    const datePublished = primary?.datePublished;
+    if (datePublished) lines.push(`datePublished: "${escapeYaml(datePublished)}"`);
+    const dateModified = primary?.dateModified;
+    if (dateModified) lines.push(`dateModified: "${escapeYaml(dateModified)}"`);
+  }
+  lines.push("---");
+  return lines.join("\n");
+}
+function escapeYaml(str) {
+  return str.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, " ");
+}
+
+// src/transform.ts
+async function transform(html, options = {}) {
+  const {
+    url,
+    tokenBudget,
+    extractJsonLd = true,
+    stripSelectors = [],
+    preserveSelectors = [],
+    frontmatter = true,
+    turndownRules = []
+  } = options;
+  const dom = new JSDOM2(html, { url: url || "https://localhost" });
+  const document = dom.window.document;
+  const title = document.querySelector("title")?.textContent?.trim() || document.querySelector("h1")?.textContent?.trim() || "";
+  const description = document.querySelector('meta[name="description"]')?.getAttribute("content")?.trim() || "";
+  const canonicalUrl = document.querySelector('link[rel="canonical"]')?.getAttribute("href") || null;
+  const lang = document.documentElement.getAttribute("lang") || null;
+  const lastModified = document.querySelector('meta[property="article:modified_time"]')?.getAttribute("content") || document.querySelector('meta[name="last-modified"]')?.getAttribute("content") || null;
+  const jsonLd = extractJsonLd ? extractJsonLdBlocks(document) : [];
+  let contentHtml;
+  if (isProbablyReaderable(document)) {
+    const reader = new Readability(document, { charThreshold: 100 });
+    const article = reader.parse();
+    contentHtml = article?.content || document.body?.innerHTML || html;
+  } else {
+    contentHtml = document.body?.innerHTML || html;
+  }
+  const cleanHtml = sanitizeHtml(contentHtml, {
+    stripSelectors,
+    preserveSelectors
+  });
+  let markdown = htmlToMarkdown(cleanHtml, { url, customRules: turndownRules });
+  if (frontmatter) {
+    const fm = buildFrontmatter({ title, description, url, lang, lastModified, jsonLd });
+    markdown = fm + "\n\n" + markdown;
+  }
+  let tokenEstimate = estimateTokens(markdown);
+  if (tokenBudget && tokenEstimate > tokenBudget) {
+    markdown = truncateToTokenBudget(markdown, tokenBudget);
+    tokenEstimate = estimateTokens(markdown);
+  }
+  dom.window.close();
+  return {
+    markdown,
+    tokenEstimate,
+    title,
+    description,
+    jsonLd,
+    canonicalUrl,
+    lastModified,
+    lang
+  };
+}
+function truncateToTokenBudget(markdown, budget) {
+  const lines = markdown.split("\n");
+  const result = [];
+  let currentTokens = 0;
+  for (const line of lines) {
+    const lineTokens = estimateTokens(line);
+    if (currentTokens + lineTokens > budget) {
+      if (/^#{1,6}\s/.test(line)) {
+        result.push(line);
+        result.push("\n*[Content truncated for token budget]*\n");
+      }
+      break;
+    }
+    result.push(line);
+    currentTokens += lineTokens;
+  }
+  return result.join("\n");
+}
+
+// src/llms-txt.ts
+function generateLlmsTxt(options, routes, fullTextContents) {
+  const { siteName, siteDescription, baseUrl, markdownExtension = ".md" } = options;
+  const sections = /* @__PURE__ */ new Map();
+  for (const route of routes) {
+    const section = route.section || "Pages";
+    if (!sections.has(section)) sections.set(section, []);
+    sections.get(section).push(route);
+  }
+  const lines = [];
+  lines.push(`# ${siteName}`);
+  lines.push("");
+  lines.push(`> ${siteDescription}`);
+  lines.push("");
+  for (const [section, sectionRoutes] of sections) {
+    lines.push(`## ${section}`);
+    lines.push("");
+    for (const route of sectionRoutes) {
+      const url = `${baseUrl}${route.path}${markdownExtension}`;
+      const desc = route.description ? `: ${route.description}` : "";
+      lines.push(`- [${route.title}](${url})${desc}`);
+    }
+    lines.push("");
+  }
+  const llmsTxt = lines.join("\n").trim() + "\n";
+  const fullLines = [];
+  fullLines.push(`# ${siteName}`);
+  fullLines.push("");
+  fullLines.push(`> ${siteDescription}`);
+  fullLines.push("");
+  if (fullTextContents) {
+    for (const route of routes) {
+      const content = fullTextContents.get(route.path);
+      if (content) {
+        fullLines.push(`---`);
+        fullLines.push("");
+        fullLines.push(`## ${route.title}`);
+        fullLines.push(`Source: ${baseUrl}${route.path}`);
+        fullLines.push("");
+        fullLines.push(content);
+        fullLines.push("");
+      }
+    }
+  }
+  const llmsFullTxt = fullLines.join("\n").trim() + "\n";
+  return {
+    llmsTxt,
+    llmsFullTxt,
+    routeCount: routes.length
+  };
+}
+
+// src/discover.ts
+import { readFileSync, existsSync, readdirSync, lstatSync, realpathSync } from "fs";
+import { join, sep } from "path";
+var PAGE_FILE_PATTERNS = [
+  "page.tsx",
+  "page.ts",
+  "page.jsx",
+  "page.js",
+  "page.mdx",
+  "page.md"
+];
+var SKIP_DIRS = /* @__PURE__ */ new Set([
+  "node_modules",
+  ".next",
+  ".git",
+  "dist",
+  ".turbo",
+  "_components",
+  "_lib",
+  "_utils",
+  "_hooks",
+  "_actions",
+  "components",
+  "lib",
+  "utils",
+  "hooks",
+  "actions",
+  "api"
+  // API routes are not pages
+]);
+function discoverNextRoutes(appDir, options = {}) {
+  const {
+    exclude = ["/api"],
+    sectionStrategy = "directory",
+    defaultSection = "Pages"
+  } = options;
+  if (!existsSync(appDir)) {
+    return [];
+  }
+  const routes = [];
+  let rootReal = appDir;
+  try {
+    rootReal = realpathSync(appDir);
+  } catch {
+    return routes;
+  }
+  scanAppDir(appDir, appDir, rootReal, routes, exclude, sectionStrategy, defaultSection);
+  routes.sort((a, b) => {
+    if (a.path === "/") return -1;
+    if (b.path === "/") return 1;
+    return a.path.localeCompare(b.path);
+  });
+  return routes;
+}
+function scanAppDir(rootDir, currentDir, rootReal, routes, exclude, sectionStrategy, defaultSection) {
+  const entries = readdirSync(currentDir);
+  for (const entry of entries) {
+    const fullPath = join(currentDir, entry);
+    let stat;
+    try {
+      stat = lstatSync(fullPath);
+    } catch {
+      continue;
+    }
+    if (stat.isSymbolicLink()) continue;
+    if (stat.isDirectory()) {
+      let realDir = fullPath;
+      try {
+        realDir = realpathSync(fullPath);
+      } catch {
+        continue;
+      }
+      if (!isWithinRoot(realDir, rootReal)) continue;
+      const baseName = entry.toLowerCase();
+      if (baseName.startsWith("_") || baseName.startsWith(".") || SKIP_DIRS.has(baseName))
+        continue;
+      scanAppDir(
+        rootDir,
+        fullPath,
+        rootReal,
+        routes,
+        exclude,
+        sectionStrategy,
+        defaultSection
+      );
+      continue;
+    }
+    if (!PAGE_FILE_PATTERNS.includes(entry)) continue;
+    const relativePath = currentDir.substring(rootDir.length);
+    let urlPath = relativePath.replace(/\\/g, "/");
+    urlPath = urlPath.replace(/\/\([^)]+\)/g, "");
+    if (urlPath.includes("[...") || urlPath.includes("[[...")) continue;
+    if (urlPath === "") urlPath = "/";
+    if (!urlPath.startsWith("/")) urlPath = "/" + urlPath;
+    if (shouldExclude(urlPath, exclude)) continue;
+    if (urlPath === "/llms.txt" || urlPath === "/llms-full.txt") continue;
+    const { title, description } = extractMetadataFromFile(fullPath);
+    const section = deriveSection(urlPath, sectionStrategy, defaultSection);
+    routes.push({
+      path: urlPath,
+      title: title || pathToTitle(urlPath),
+      description: description || void 0,
+      section
+    });
+  }
+}
+function extractMetadataFromFile(filePath) {
+  try {
+    const content = readFileSync(filePath, "utf-8");
+    return extractMetadataFromSource(content);
+  } catch {
+    return { title: "", description: "" };
+  }
+}
+function extractMetadataFromSource(source) {
+  let title = "";
+  let description = "";
+  const metadataMatch = source.match(
+    /export\s+const\s+metadata[\s:]*(?:Metadata\s*)?=\s*\{/
+  );
+  if (!metadataMatch) {
+    return { title, description };
+  }
+  const startIdx = metadataMatch.index + metadataMatch[0].length - 1;
+  const objectStr = extractBalancedBraces(source, startIdx);
+  if (!objectStr) {
+    return { title, description };
+  }
+  const titleMatch = objectStr.match(
+    /title\s*:\s*(?:'([^']*)'|"([^"]*)"|`([^`]*)`)/
+  );
+  if (titleMatch) {
+    title = sanitizeMetadataValue(titleMatch[1] || titleMatch[2] || titleMatch[3] || "");
+  }
+  const descMatch = objectStr.match(
+    /description\s*:\s*(?:'([^']*)'|"([^"]*)"|`([^`]*)`)/
+  );
+  if (descMatch) {
+    description = sanitizeMetadataValue(descMatch[1] || descMatch[2] || descMatch[3] || "");
+  }
+  return { title, description };
+}
+function extractBalancedBraces(source, start) {
+  if (source[start] !== "{") return null;
+  let depth = 0;
+  for (let i = start; i < source.length; i++) {
+    if (source[i] === "{") depth++;
+    else if (source[i] === "}") depth--;
+    if (depth === 0) {
+      return source.substring(start, i + 1);
+    }
+  }
+  return null;
+}
+function deriveSection(urlPath, strategy, defaultSection) {
+  if (typeof strategy === "function") {
+    return strategy(urlPath);
+  }
+  const segments = urlPath.split("/").filter(Boolean);
+  if (segments.length === 0) return defaultSection;
+  const firstSegment = segments[0];
+  if (firstSegment.startsWith("[")) return defaultSection;
+  return firstSegment.split("-").map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
+}
+function pathToTitle(urlPath) {
+  if (urlPath === "/") return "Home";
+  const lastSegment = urlPath.split("/").filter(Boolean).pop() || "";
+  if (lastSegment.startsWith("[")) {
+    return lastSegment.replace(/^\[|\]$/g, "");
+  }
+  return lastSegment.split("-").map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
+}
+function shouldExclude(urlPath, patterns) {
+  for (const pattern of patterns) {
+    if (pattern.endsWith("/**") || pattern.endsWith("/*")) {
+      const prefix = pattern.replace(/\/\*\*?$/, "");
+      if (urlPath === prefix || urlPath.startsWith(prefix + "/")) return true;
+    } else if (pattern.startsWith("_")) {
+      if (urlPath.startsWith("/" + pattern) || urlPath.includes("/" + pattern + "/"))
+        return true;
+    } else if (urlPath === pattern || urlPath.startsWith(pattern + "/")) {
+      return true;
+    }
+  }
+  return false;
+}
+function discoverFilesystemRoutes(dir, options = {}) {
+  const {
+    exclude = [],
+    sectionStrategy = "directory",
+    defaultSection = "Pages"
+  } = options;
+  if (!existsSync(dir)) {
+    return [];
+  }
+  let rootReal = dir;
+  try {
+    rootReal = realpathSync(dir);
+  } catch {
+    return [];
+  }
+  const htmlFiles = findHtmlFiles(dir, rootReal);
+  const routes = [];
+  for (const filePath of htmlFiles) {
+    const relativePath = filePath.substring(dir.length);
+    let urlPath = relativePath.replace(/\\/g, "/").replace(/\/index\.html$/, "/").replace(/\.html$/, "");
+    if (urlPath === "") urlPath = "/";
+    if (!urlPath.startsWith("/")) urlPath = "/" + urlPath;
+    if (shouldExclude(urlPath, exclude)) continue;
+    const section = deriveSection(urlPath, sectionStrategy, defaultSection);
+    let title = "";
+    try {
+      const html = readFileSync(filePath, "utf-8");
+      const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+      title = sanitizeMetadataValue(titleMatch?.[1]?.trim() || "");
+    } catch {
+    }
+    routes.push({
+      path: urlPath,
+      title: title || pathToTitle(urlPath),
+      section
+    });
+  }
+  routes.sort((a, b) => {
+    if (a.path === "/") return -1;
+    if (b.path === "/") return 1;
+    return a.path.localeCompare(b.path);
+  });
+  return routes;
+}
+function findHtmlFiles(dir, rootReal) {
+  const results = [];
+  if (!existsSync(dir)) return results;
+  const entries = readdirSync(dir);
+  for (const entry of entries) {
+    const fullPath = join(dir, entry);
+    let stat;
+    try {
+      stat = lstatSync(fullPath);
+    } catch {
+      continue;
+    }
+    if (stat.isSymbolicLink()) continue;
+    if (stat.isDirectory()) {
+      if (["node_modules", ".next", ".git", "dist", ".turbo"].includes(entry))
+        continue;
+      let realDir = fullPath;
+      try {
+        realDir = realpathSync(fullPath);
+      } catch {
+        continue;
+      }
+      if (!isWithinRoot(realDir, rootReal)) continue;
+      results.push(...findHtmlFiles(fullPath, rootReal));
+    } else if (entry.endsWith(".html")) {
+      results.push(fullPath);
+    }
+  }
+  return results;
+}
+function isWithinRoot(realPath, rootReal) {
+  if (realPath === rootReal) return true;
+  const normalizedRoot = rootReal.endsWith(sep) ? rootReal : rootReal + sep;
+  return realPath.startsWith(normalizedRoot);
+}
+function sanitizeMetadataValue(value) {
+  if (!value) return "";
+  const sanitized = value.replace(/[\u0000-\u001F\u007F]/g, " ").replace(/[<>]/g, "").replace(/\s+/g, " ").trim();
+  return sanitized.length > 200 ? sanitized.slice(0, 200) : sanitized;
+}
+
+// src/headers.ts
+function buildMarkdownHeaders(result, options, originalPath) {
+  const headers = {
+    "Content-Type": "text/markdown; charset=utf-8",
+    "Content-Disposition": "inline",
+    "Vary": "Accept, User-Agent",
+    "X-Markdown-Tokens": String(result.tokenEstimate)
+  };
+  const signal = options.contentSignal ?? { aiTrain: true, search: true, aiInput: true };
+  const signalParts = [];
+  if (signal.aiTrain !== false) signalParts.push("ai-train=yes");
+  if (signal.search !== false) signalParts.push("search=yes");
+  if (signal.aiInput !== false) signalParts.push("ai-input=yes");
+  if (signalParts.length > 0) {
+    headers["Content-Signal"] = signalParts.join(", ");
+  }
+  headers["X-Robots-Tag"] = "all";
+  return headers;
+}
+function buildAlternateLinkHeader(path, ext = ".md") {
+  const mdPath = path.endsWith("/") ? `${path}index${ext}` : `${path}${ext}`;
+  return `<${mdPath}>; rel="alternate"; type="text/markdown"`;
+}
+
+// src/cache.ts
+import { LRUCache } from "lru-cache";
+function createCache(options = {}) {
+  const { maxEntries = 100, ttl = 3e5 } = options;
+  const cache = new LRUCache({
+    max: maxEntries,
+    ttl
+  });
+  return {
+    get: (key) => cache.get(key),
+    set: (key, value) => cache.set(key, value),
+    has: (key) => cache.has(key),
+    clear: () => cache.clear(),
+    size: () => cache.size
+  };
+}
+export {
+  AI_BOT_REGISTRY,
+  buildAlternateLinkHeader,
+  buildMarkdownHeaders,
+  createCache,
+  detectAgent,
+  discoverFilesystemRoutes,
+  discoverNextRoutes,
+  estimateTokens,
+  extractJsonLdBlocks,
+  extractMetadataFromSource,
+  generateLlmsTxt,
+  htmlToMarkdown,
+  sanitizeHtml,
+  shouldServeMarkdown,
+  transform
+};
+//# sourceMappingURL=index.js.map