@seoengine.ai/next-llm-ready 1.0.0

package/dist/api/index.cjs

@@ -0,0 +1,624 @@
+'use strict';
+
+var server = require('next/server');
+
+// src/api/llms-txt-handler.ts
+
+// src/server/generate-llms-txt.ts
+function generateLLMsTxt(config) {
+  const parts = [];
+  parts.push(`# ${config.siteName}`);
+  parts.push("");
+  if (config.siteDescription) {
+    parts.push(`> ${config.siteDescription}`);
+    parts.push("");
+  }
+  if (config.headerText) {
+    parts.push(config.headerText);
+  } else {
+    parts.push("This file provides LLM-friendly access to the content on this website.");
+  }
+  parts.push("");
+  parts.push("---");
+  parts.push(`- **Site URL**: ${config.siteUrl}`);
+  parts.push("- **Format**: Append `?llm=1` to any page URL to get markdown");
+  parts.push("---");
+  parts.push("");
+  if (config.content.length > 0) {
+    parts.push("## Available Content");
+    parts.push("");
+    for (const item of config.content) {
+      const llmUrl = appendQueryParam(item.url, "llm", "1");
+      parts.push(`### [${item.title}](${llmUrl})`);
+      if (item.type) {
+        parts.push(`- **Type**: ${item.type}`);
+      }
+      if (item.date) {
+        parts.push(`- **Date**: ${item.date}`);
+      }
+      if (item.description) {
+        parts.push(`- **Description**: ${item.description}`);
+      }
+      parts.push(`- **Markdown URL**: ${llmUrl}`);
+      parts.push("");
+    }
+  }
+  parts.push("---");
+  parts.push("");
+  if (config.footerText) {
+    parts.push(config.footerText);
+  } else {
+    parts.push("*Generated by [next-llm-ready](https://seoengine.ai) - Make your content AI-ready*");
+  }
+  return parts.join("\n");
+}
+function appendQueryParam(url, key, value) {
+  const separator = url.includes("?") ? "&" : "?";
+  return `${url}${separator}${key}=${value}`;
+}
+
+// src/api/llms-txt-handler.ts
+function createLLMsTxtHandler(options) {
+  return async function handler(request) {
+    try {
+      const siteConfig = await options.getSiteConfig();
+      const content = await options.getContent();
+      const config = {
+        siteName: siteConfig.siteName,
+        siteDescription: siteConfig.siteDescription,
+        siteUrl: siteConfig.siteUrl,
+        content,
+        headerText: options.headerText,
+        footerText: options.footerText
+      };
+      const llmsTxt = generateLLMsTxt(config);
+      return new server.NextResponse(llmsTxt, {
+        status: 200,
+        headers: {
+          "Content-Type": "text/plain; charset=utf-8",
+          "Cache-Control": options.cacheControl || "public, max-age=3600",
+          "X-Robots-Tag": "noindex"
+        }
+      });
+    } catch (error) {
+      console.error("Error generating llms.txt:", error);
+      return new server.NextResponse("Error generating llms.txt", { status: 500 });
+    }
+  };
+}
+function createLLMsTxtPageHandler(options) {
+  return async function handler(req, res) {
+    if (req.method !== "GET") {
+      res.status(405).end("Method Not Allowed");
+      return;
+    }
+    try {
+      const siteConfig = await options.getSiteConfig();
+      const content = await options.getContent();
+      const config = {
+        siteName: siteConfig.siteName,
+        siteDescription: siteConfig.siteDescription,
+        siteUrl: siteConfig.siteUrl,
+        content,
+        headerText: options.headerText,
+        footerText: options.footerText
+      };
+      const llmsTxt = generateLLMsTxt(config);
+      res.setHeader("Content-Type", "text/plain; charset=utf-8");
+      res.setHeader("Cache-Control", options.cacheControl || "public, max-age=3600");
+      res.setHeader("X-Robots-Tag", "noindex");
+      res.status(200).end(llmsTxt);
+    } catch (error) {
+      console.error("Error generating llms.txt:", error);
+      res.status(500).end("Error generating llms.txt");
+    }
+  };
+}
+
+// src/utils/html-to-markdown.ts
+var defaultOptions = {
+  preserveLineBreaks: true,
+  convertImages: true,
+  convertLinks: true,
+  stripScripts: true,
+  customHandlers: {}
+};
+function htmlToMarkdown(html, options = {}) {
+  const opts = { ...defaultOptions, ...options };
+  if (typeof window === "undefined") {
+    return serverSideConvert(html, opts);
+  }
+  const doc = new DOMParser().parseFromString(html, "text/html");
+  return convertNode(doc.body, opts);
+}
+function serverSideConvert(html, opts) {
+  let markdown = html;
+  if (opts.stripScripts) {
+    markdown = markdown.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "");
+    markdown = markdown.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "");
+  }
+  markdown = markdown.replace(/<h1[^>]*>(.*?)<\/h1>/gi, "\n# $1\n");
+  markdown = markdown.replace(/<h2[^>]*>(.*?)<\/h2>/gi, "\n## $1\n");
+  markdown = markdown.replace(/<h3[^>]*>(.*?)<\/h3>/gi, "\n### $1\n");
+  markdown = markdown.replace(/<h4[^>]*>(.*?)<\/h4>/gi, "\n#### $1\n");
+  markdown = markdown.replace(/<h5[^>]*>(.*?)<\/h5>/gi, "\n##### $1\n");
+  markdown = markdown.replace(/<h6[^>]*>(.*?)<\/h6>/gi, "\n###### $1\n");
+  markdown = markdown.replace(/<(strong|b)[^>]*>(.*?)<\/\1>/gi, "**$2**");
+  markdown = markdown.replace(/<(em|i)[^>]*>(.*?)<\/\1>/gi, "*$2*");
+  if (opts.convertLinks) {
+    markdown = markdown.replace(/<a[^>]*href=["']([^"']*)["'][^>]*>(.*?)<\/a>/gi, "[$2]($1)");
+  }
+  if (opts.convertImages) {
+    markdown = markdown.replace(
+      /<img[^>]*src=["']([^"']*)["'][^>]*alt=["']([^"']*)["'][^>]*\/?>/gi,
+      "![$2]($1)"
+    );
+    markdown = markdown.replace(
+      /<img[^>]*alt=["']([^"']*)["'][^>]*src=["']([^"']*)["'][^>]*\/?>/gi,
+      "![$1]($2)"
+    );
+    markdown = markdown.replace(/<img[^>]*src=["']([^"']*)["'][^>]*\/?>/gi, "![]($1)");
+  }
+  markdown = markdown.replace(/<ul[^>]*>/gi, "\n");
+  markdown = markdown.replace(/<\/ul>/gi, "\n");
+  markdown = markdown.replace(/<ol[^>]*>/gi, "\n");
+  markdown = markdown.replace(/<\/ol>/gi, "\n");
+  markdown = markdown.replace(/<li[^>]*>(.*?)<\/li>/gi, "- $1\n");
+  markdown = markdown.replace(/<blockquote[^>]*>(.*?)<\/blockquote>/gis, (_, content) => {
+    return "\n> " + content.trim().replace(/\n/g, "\n> ") + "\n";
+  });
+  markdown = markdown.replace(/<pre[^>]*><code[^>]*>(.*?)<\/code><\/pre>/gis, "\n```\n$1\n```\n");
+  markdown = markdown.replace(/<code[^>]*>(.*?)<\/code>/gi, "`$1`");
+  markdown = markdown.replace(/<p[^>]*>(.*?)<\/p>/gis, "\n$1\n");
+  markdown = markdown.replace(/<br\s*\/?>/gi, "\n");
+  markdown = markdown.replace(/<hr[^>]*\/?>/gi, "\n---\n");
+  markdown = markdown.replace(/<[^>]+>/g, "");
+  markdown = decodeHTMLEntities(markdown);
+  markdown = markdown.replace(/\n{3,}/g, "\n\n");
+  markdown = markdown.trim();
+  return markdown;
+}
+function convertNode(node, opts) {
+  if (node.nodeType === Node.TEXT_NODE) {
+    return node.textContent || "";
+  }
+  if (node.nodeType !== Node.ELEMENT_NODE) {
+    return "";
+  }
+  const element = node;
+  const tagName = element.tagName.toLowerCase();
+  if (opts.customHandlers?.[tagName]) {
+    return opts.customHandlers[tagName](element);
+  }
+  if (opts.stripScripts && (tagName === "script" || tagName === "style")) {
+    return "";
+  }
+  const childContent = Array.from(element.childNodes).map((child) => convertNode(child, opts)).join("");
+  switch (tagName) {
+    case "h1":
+      return `
+# ${childContent.trim()}
+`;
+    case "h2":
+      return `
+## ${childContent.trim()}
+`;
+    case "h3":
+      return `
+### ${childContent.trim()}
+`;
+    case "h4":
+      return `
+#### ${childContent.trim()}
+`;
+    case "h5":
+      return `
+##### ${childContent.trim()}
+`;
+    case "h6":
+      return `
+###### ${childContent.trim()}
+`;
+    case "p":
+      return `
+${childContent.trim()}
+`;
+    case "br":
+      return opts.preserveLineBreaks ? "\n" : " ";
+    case "strong":
+    case "b":
+      return `**${childContent}**`;
+    case "em":
+    case "i":
+      return `*${childContent}*`;
+    case "u":
+      return `_${childContent}_`;
+    case "s":
+    case "strike":
+    case "del":
+      return `~~${childContent}~~`;
+    case "a":
+      if (opts.convertLinks) {
+        const href = element.getAttribute("href") || "";
+        return `[${childContent}](${href})`;
+      }
+      return childContent;
+    case "img":
+      if (opts.convertImages) {
+        const src = element.getAttribute("src") || "";
+        const alt = element.getAttribute("alt") || "";
+        return `![${alt}](${src})`;
+      }
+      return "";
+    case "ul":
+      return `
+${childContent}
+`;
+    case "ol":
+      return `
+${childContent}
+`;
+    case "li":
+      return `- ${childContent.trim()}
+`;
+    case "blockquote":
+      return `
+> ${childContent.trim().replace(/\n/g, "\n> ")}
+`;
+    case "pre":
+      const codeElement = element.querySelector("code");
+      const lang = codeElement?.className.match(/language-(\w+)/)?.[1] || "";
+      const code = codeElement?.textContent || childContent;
+      return `
+\`\`\`${lang}
+${code.trim()}
+\`\`\`
+`;
+    case "code":
+      if (element.parentElement?.tagName.toLowerCase() === "pre") {
+        return childContent;
+      }
+      return `\`${childContent}\``;
+    case "hr":
+      return "\n---\n";
+    case "table":
+      return convertTable(element);
+    case "div":
+    case "section":
+    case "article":
+    case "main":
+    case "aside":
+    case "header":
+    case "footer":
+    case "nav":
+      return childContent;
+    default:
+      return childContent;
+  }
+}
+function convertTable(table) {
+  const rows = table.querySelectorAll("tr");
+  if (rows.length === 0) return "";
+  let markdown = "\n";
+  let headerProcessed = false;
+  rows.forEach((row, index) => {
+    const cells = row.querySelectorAll("th, td");
+    const rowContent = Array.from(cells).map((cell) => cell.textContent?.trim() || "").join(" | ");
+    markdown += `| ${rowContent} |
+`;
+    if (!headerProcessed && (row.querySelector("th") || index === 0)) {
+      const separator = Array.from(cells).map(() => "---").join(" | ");
+      markdown += `| ${separator} |
+`;
+      headerProcessed = true;
+    }
+  });
+  return markdown + "\n";
+}
+function decodeHTMLEntities(text) {
+  const entities = {
+    "&amp;": "&",
+    "&lt;": "<",
+    "&gt;": ">",
+    "&quot;": '"',
+    "&#39;": "'",
+    "&apos;": "'",
+    "&nbsp;": " ",
+    "&mdash;": "\u2014",
+    "&ndash;": "\u2013",
+    "&hellip;": "\u2026",
+    "&copy;": "\xA9",
+    "&reg;": "\xAE",
+    "&trade;": "\u2122"
+  };
+  let result = text;
+  for (const [entity, char] of Object.entries(entities)) {
+    result = result.replace(new RegExp(entity, "g"), char);
+  }
+  result = result.replace(/&#(\d+);/g, (_, num) => String.fromCharCode(parseInt(num, 10)));
+  result = result.replace(
+    /&#x([0-9a-f]+);/gi,
+    (_, hex) => String.fromCharCode(parseInt(hex, 16))
+  );
+  return result;
+}
+function countWords(text) {
+  return text.replace(/[^\w\s]/g, "").split(/\s+/).filter((word) => word.length > 0).length;
+}
+function calculateReadingTime(text, wordsPerMinute = 200) {
+  const words = countWords(text);
+  return Math.ceil(words / wordsPerMinute);
+}
+
+// src/server/generate-markdown.ts
+function generateMarkdown(content) {
+  const parts = [];
+  if (content.promptPrefix?.trim()) {
+    parts.push(content.promptPrefix.trim());
+    parts.push("");
+  }
+  parts.push(`# ${content.title}`);
+  parts.push("");
+  if (content.excerpt) {
+    parts.push(`> ${content.excerpt}`);
+    parts.push("");
+  }
+  parts.push("---");
+  parts.push(`- **Source**: ${content.url}`);
+  if (content.date) {
+    parts.push(`- **Date**: ${content.date}`);
+  }
+  if (content.modifiedDate) {
+    parts.push(`- **Modified**: ${content.modifiedDate}`);
+  }
+  if (content.author) {
+    parts.push(`- **Author**: ${content.author}`);
+  }
+  if (content.categories?.length) {
+    parts.push(`- **Categories**: ${content.categories.join(", ")}`);
+  }
+  if (content.tags?.length) {
+    parts.push(`- **Tags**: ${content.tags.join(", ")}`);
+  }
+  if (content.readingTime) {
+    parts.push(`- **Reading Time**: ${content.readingTime} min`);
+  }
+  parts.push("---");
+  parts.push("");
+  if (content.content) {
+    const contentMarkdown = isHTML(content.content) ? htmlToMarkdown(content.content) : content.content;
+    parts.push(contentMarkdown);
+  }
+  const markdown = parts.join("\n").trim();
+  const wordCount = countWords(markdown);
+  const readingTime = calculateReadingTime(markdown);
+  const headings = extractMarkdownHeadings(markdown);
+  return {
+    markdown,
+    wordCount,
+    readingTime,
+    headings
+  };
+}
+function generateMarkdownString(content) {
+  return generateMarkdown(content).markdown;
+}
+function isHTML(str) {
+  return /<[a-z][\s\S]*>/i.test(str);
+}
+function extractMarkdownHeadings(markdown) {
+  const headings = [];
+  const lines = markdown.split("\n");
+  let index = 0;
+  for (const line of lines) {
+    const match = line.match(/^(#{1,6})\s+(.+)$/);
+    if (match) {
+      const level = match[1].length;
+      const text = match[2].trim();
+      const id = generateSlug(text, index);
+      headings.push({ id, text, level });
+      index++;
+    }
+  }
+  return headings;
+}
+function generateSlug(text, index) {
+  const slug = text.toLowerCase().replace(/[^a-z0-9\s-]/g, "").replace(/\s+/g, "-").replace(/-+/g, "-").replace(/^-+|-+$/g, "");
+  return slug || `heading-${index}`;
+}
+
+// src/api/markdown-handler.ts
+function withLLMParam(options) {
+  return async function middleware(request) {
+    const url = new URL(request.url);
+    const llmParam = url.searchParams.get("llm");
+    if (llmParam !== "1") {
+      return void 0;
+    }
+    try {
+      const pathname = url.pathname;
+      const content = await options.getContent(pathname);
+      if (!content) {
+        return new server.NextResponse("Content not found", { status: 404 });
+      }
+      const markdown = generateMarkdownString(content);
+      return new server.NextResponse(markdown, {
+        status: 200,
+        headers: {
+          "Content-Type": "text/markdown; charset=utf-8",
+          "Cache-Control": options.cacheControl || "public, max-age=3600",
+          "X-Robots-Tag": "noindex",
+          ...options.cors && {
+            "Access-Control-Allow-Origin": "*",
+            "Access-Control-Allow-Methods": "GET"
+          }
+        }
+      });
+    } catch (error) {
+      console.error("Error generating markdown:", error);
+      return new server.NextResponse("Error generating markdown", { status: 500 });
+    }
+  };
+}
+function createMarkdownHandler(options) {
+  return async function handler(request, { params }) {
+    try {
+      const slug = params?.slug || "";
+      const content = await options.getContent(slug);
+      if (!content) {
+        return new server.NextResponse("Content not found", {
+          status: 404,
+          headers: { "Content-Type": "text/plain" }
+        });
+      }
+      const markdown = generateMarkdownString(content);
+      return new server.NextResponse(markdown, {
+        status: 200,
+        headers: {
+          "Content-Type": "text/markdown; charset=utf-8",
+          "Cache-Control": options.cacheControl || "public, max-age=3600",
+          "X-Robots-Tag": "noindex",
+          ...options.cors && {
+            "Access-Control-Allow-Origin": "*",
+            "Access-Control-Allow-Methods": "GET"
+          }
+        }
+      });
+    } catch (error) {
+      console.error("Error generating markdown:", error);
+      return new server.NextResponse("Error generating markdown", { status: 500 });
+    }
+  };
+}
+function hasLLMParam(request) {
+  const url = new URL(request.url);
+  return url.searchParams.get("llm") === "1";
+}
+function createInMemoryStorage() {
+  const events = [];
+  return {
+    save: async (event) => {
+      events.push(event);
+    },
+    getAll: async () => [...events],
+    clear: async () => {
+      events.length = 0;
+    }
+  };
+}
+function createAnalyticsHandler(options) {
+  const requestCounts = /* @__PURE__ */ new Map();
+  return async function handler(request) {
+    if (request.method === "OPTIONS") {
+      return new server.NextResponse(null, {
+        status: 204,
+        headers: {
+          "Access-Control-Allow-Origin": "*",
+          "Access-Control-Allow-Methods": "POST, OPTIONS",
+          "Access-Control-Allow-Headers": "Content-Type"
+        }
+      });
+    }
+    if (request.method !== "POST") {
+      return new server.NextResponse("Method Not Allowed", { status: 405 });
+    }
+    if (options.rateLimit) {
+      const clientIp = request.headers.get("x-forwarded-for") || "unknown";
+      const now = Date.now();
+      const windowMs = 6e4;
+      const clientData = requestCounts.get(clientIp) || { count: 0, resetAt: now + windowMs };
+      if (now > clientData.resetAt) {
+        clientData.count = 0;
+        clientData.resetAt = now + windowMs;
+      }
+      clientData.count++;
+      requestCounts.set(clientIp, clientData);
+      if (clientData.count > options.rateLimit) {
+        return new server.NextResponse("Rate limit exceeded", {
+          status: 429,
+          headers: {
+            "Retry-After": Math.ceil((clientData.resetAt - now) / 1e3).toString()
+          }
+        });
+      }
+    }
+    try {
+      const body = await request.json();
+      const event = {
+        action: body.action || "copy",
+        contentId: body.contentId,
+        url: body.url || request.headers.get("referer"),
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        metadata: body.metadata
+      };
+      await options.storage.save(event);
+      const headers = {
+        "Content-Type": "application/json"
+      };
+      if (options.cors) {
+        headers["Access-Control-Allow-Origin"] = "*";
+      }
+      return server.NextResponse.json({ success: true, event }, { headers });
+    } catch (error) {
+      console.error("Analytics tracking error:", error);
+      return server.NextResponse.json({ success: false, error: "Failed to track event" }, { status: 500 });
+    }
+  };
+}
+function createAnalyticsPageHandler(options) {
+  return async function handler(req, res) {
+    if (req.method !== "POST") {
+      res.status(405).end("Method Not Allowed");
+      return;
+    }
+    try {
+      const body = req.body;
+      const event = {
+        action: body?.action || "copy",
+        contentId: body?.contentId,
+        url: body?.url || req.headers?.referer,
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        metadata: body?.metadata
+      };
+      await options.storage.save(event);
+      if (options.cors) {
+        res.setHeader("Access-Control-Allow-Origin", "*");
+      }
+      res.status(200).json({ success: true, event });
+    } catch (error) {
+      console.error("Analytics tracking error:", error);
+      res.status(500).json({ success: false, error: "Failed to track event" });
+    }
+  };
+}
+async function aggregateEvents(storage) {
+  const events = await storage.getAll();
+  const counts = {
+    copy: 0,
+    view: 0,
+    download: 0
+  };
+  for (const event of events) {
+    if (event.action in counts) {
+      counts[event.action]++;
+    }
+  }
+  return counts;
+}
+async function getEventsForContent(storage, contentId) {
+  const events = await storage.getAll();
+  return events.filter((event) => event.contentId === contentId);
+}
+
+exports.aggregateEvents = aggregateEvents;
+exports.createAnalyticsHandler = createAnalyticsHandler;
+exports.createAnalyticsPageHandler = createAnalyticsPageHandler;
+exports.createInMemoryStorage = createInMemoryStorage;
+exports.createLLMsTxtHandler = createLLMsTxtHandler;
+exports.createLLMsTxtPageHandler = createLLMsTxtPageHandler;
+exports.createMarkdownHandler = createMarkdownHandler;
+exports.getEventsForContent = getEventsForContent;
+exports.hasLLMParam = hasLLMParam;
+exports.withLLMParam = withLLMParam;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map