@happyalienai/vite-plugin-llm-spider 0.1.0 → 0.2.0

package/README.md

@@ -1,10 +1,20 @@
 # vite-plugin-llm-spider
 
-> **Built by [Happy Alien AI](https://happyalien.ai)** — AI-powered tools for eLearning creators.
+> Make your Vite SPA discoverable by AI search engines like ChatGPT, Perplexity, and Google AI Overviews
 
-A Vite build plugin that generates **LLM-friendly Markdown snapshots** of selected public routes and publishes a curated index at **`/llms.txt`**.
+**Single Page Apps are invisible to AI.** While tools like ChatGPT, Claude, and Perplexity reshape how people find information, SPAs remain hidden behind JavaScript walls. This plugin bridges that gap by generating clean, LLM-friendly markdown snapshots and a standardized index.
 
-Makes SPAs and content-heavy Vite apps easier for AI agents/tools to understand by providing clean, low-noise text renditions plus a deterministic index.
+- 🤖 **Zero-click optimization** — Get cited in AI answers without complex SSR
+- 📈 **LLM SEO / GEO ready** — Clean markdown format that LLMs prefer
+- 🎯 **[llms.txt standard](https://llmstxt.org/)** — Machine-readable index for AI agents
+
+> **Built by [Happy Alien AI](https://happyalien.ai)** — We take the busy work out of training development.
+
+## Why This Matters
+
+Traditional SEO optimizes for Google's crawler. **Generative Engine Optimization (GEO)** optimizes for AI systems that synthesize answers from your content. When someone asks ChatGPT or Perplexity a question your site answers, you want to be cited.
+
+SPAs render content via JavaScript — invisible to most AI crawlers. This plugin runs Puppeteer at build time to capture your rendered pages as clean markdown, plus generates an `llms.txt` index that tells AI agents exactly where to look.
 
 ## Features
 
@@ -72,6 +82,29 @@ The generated `llms.txt` follows the [llmstxt.org](https://llmstxt.org/) spec:
 
 ## Configuration
 
+### Static Mode (No Browser Required)
+
+By default, when crawl is disabled, the plugin reads HTML files directly from your `dist/` folder — **no Puppeteer or browser needed**. This works great for:
+
+- Pre-rendered/SSG sites
+- CI environments without Chrome (WSL, Docker, etc.)
+- Simple static sites
+
+```js
+llmSpider({
+  static: true,  // Force static mode (default: "auto")
+  routes: [
+    { path: "/", title: "Home" },
+    { path: "/about", title: "About" },
+  ],
+})
+```
+
+Options:
+- `true` — Always read HTML files directly (no browser)
+- `false` — Always use Puppeteer (required for SPAs that need JS rendering)
+- `"auto"` (default) — Use static when crawl is disabled, browser when crawl is enabled
+
 ### Route Definitions
 
 ```js
@@ -207,6 +240,7 @@ render: {
 - Add more `removeSelectors`
 
 ### CI fails to launch browser
+- **Easiest fix:** Use `static: true` if you don't need JS rendering
 - Add `--no-sandbox` to launch args
 - Ensure Puppeteer dependencies are installed
 
@@ -221,5 +255,5 @@ MIT
     <strong>Happy Alien AI</strong>
   </a>
   <br>
-  AI-powered tools for instructional designers and eLearning teams
+  We take the busy work out of training development.
 </p>

package/dist/index.cjs

@@ -39,7 +39,13 @@ var import_node_path = __toESM(require("path"), 1);
 var cheerio = __toESM(require("cheerio"), 1);
 var import_turndown = __toESM(require("turndown"), 1);
 var import_turndown_plugin_gfm = require("turndown-plugin-gfm");
-var import_puppeteer = __toESM(require("puppeteer"), 1);
+var puppeteer = null;
+async function loadPuppeteer() {
+  if (!puppeteer) {
+    puppeteer = await import("puppeteer");
+  }
+  return puppeteer.default || puppeteer;
+}
 function llmSpiderPlugin(userOptions = {}) {
   let resolvedConfig;
   function deepMerge(target, source) {
@@ -55,6 +61,11 @@ function llmSpiderPlugin(userOptions = {}) {
   }
   const defaults = {
     enabled: true,
+    // Static mode: read HTML files directly from dist/ without browser
+    // - true: always use static mode (no Puppeteer)
+    // - false: always use browser rendering
+    // - "auto" (default): use static when crawl is disabled, browser when crawl is enabled
+    static: "auto",
     // Recommended: explicit list
     routes: (
       /** @type {RouteDef[] | undefined} */
@@ -185,6 +196,11 @@ function llmSpiderPlugin(userOptions = {}) {
     }
     return import_node_path.default.join(distDir, rel);
   }
+  function routeToHtmlFsPath(distDir, route) {
+    if (route === "/") return import_node_path.default.join(distDir, "index.html");
+    if (route.endsWith("/")) return import_node_path.default.join(distDir, route.slice(1), "index.html");
+    return import_node_path.default.join(distDir, route.slice(1) + ".html");
+  }
   function makeLlmsLink(relMdPath) {
     return relMdPath.replace(/\\/g, "/");
   }
@@ -193,6 +209,12 @@ function llmSpiderPlugin(userOptions = {}) {
       server.close((err) => err ? reject(err) : resolve());
     });
   }
+  function shouldUseStaticMode() {
+    var _a;
+    if (options.static === true) return true;
+    if (options.static === false) return false;
+    return !((_a = options.crawl) == null ? void 0 : _a.enabled);
+  }
   return {
     name: "vite-plugin-llm-spider",
     apply: "build",
@@ -206,6 +228,7 @@ function llmSpiderPlugin(userOptions = {}) {
         throw new Error("LLM Spider: missing resolved Vite config");
       const distDir = resolvedConfig.build.outDir || "dist";
       const basePath = (resolvedConfig.base || "/").replace(/\\/g, "/");
+      const useStaticMode = shouldUseStaticMode();
       let routeDefs = [];
       if (Array.isArray(options.routes) && options.routes.length) {
         routeDefs = options.routes.map((r) => ({
@@ -220,98 +243,41 @@ function llmSpiderPlugin(userOptions = {}) {
       } else {
         routeDefs = [{ path: "/", section: "Pages" }];
       }
-      log.info("\nLLM Spider: generating markdown + llms.txt");
+      log.info(`
+LLM Spider: generating markdown + llms.txt (${useStaticMode ? "static" : "browser"} mode)`);
       log.debug("distDir:", distDir, "base:", basePath);
-      const previewServer = await (0, import_vite.preview)({
-        root: resolvedConfig.root,
-        base: resolvedConfig.base,
-        build: { outDir: distDir },
-        preview: { port: 0, open: false, host: "127.0.0.1" },
-        configFile: false,
-        plugins: [],
-        // avoid loading user plugins again
-        logLevel: "silent"
-      });
-      await new Promise((resolve, reject) => {
-        const server = previewServer.httpServer;
-        if (server.listening) {
-          resolve();
-        } else {
-          server.once("listening", resolve);
-          server.once("error", reject);
-          setTimeout(() => reject(new Error("Preview server failed to start")), 5e3);
-        }
-      });
-      const addr = previewServer.httpServer.address();
-      if (!addr || typeof addr === "string") {
-        await safeCloseHttpServer(previewServer.httpServer);
-        throw new Error("LLM Spider: could not determine preview server port");
-      }
-      const normalizedBase = basePath.endsWith("/") ? basePath : basePath + "/";
-      const baseUrl = `http://127.0.0.1:${addr.port}${normalizedBase}`;
-      log.debug("Preview server at:", baseUrl);
-      const browser = await import_puppeteer.default.launch(options.render.launchOptions);
       const turndown = new import_turndown.default(options.markdown.turndown);
       turndown.use(import_turndown_plugin_gfm.gfm);
-      const visited = /* @__PURE__ */ new Set();
       const captured = [];
-      const queue = [];
-      if ((_b = options.crawl) == null ? void 0 : _b.enabled) {
-        for (const seed of options.crawl.seeds || ["/"]) {
-          const nr = normalizeRoute(seed, {
-            stripQuery: options.crawl.stripQuery
-          });
-          if (nr) queue.push({ route: nr, depth: 0 });
-        }
-      } else {
-        for (const rd of routeDefs) queue.push({ route: rd.path, depth: 0 });
-      }
-      const maxDepth = ((_c = options.crawl) == null ? void 0 : _c.enabled) ? options.crawl.maxDepth : 0;
-      const maxPages = ((_d = options.crawl) == null ? void 0 : _d.enabled) ? options.crawl.maxPages : queue.length;
-      const concurrency = ((_e = options.crawl) == null ? void 0 : _e.enabled) ? options.crawl.concurrency : 3;
-      async function captureOne(route) {
-        var _a2, _b2, _c2;
-        if (visited.has(route)) return;
-        if (isExcluded(route)) return;
-        if (captured.length >= maxPages) return;
-        visited.add(route);
-        const page = await browser.newPage();
-        if ((_a2 = options.render.blockRequests) == null ? void 0 : _a2.length) {
-          await page.setRequestInterception(true);
-          page.on("request", (req) => {
-            const url = req.url();
-            const blocked = options.render.blockRequests.some(
-              (p) => p instanceof RegExp ? p.test(url) : url.includes(p)
-            );
-            if (blocked) req.abort();
-            else req.continue();
-          });
-        }
-        try {
-          const pageUrl = route === "/" ? baseUrl : baseUrl + route.replace(/^\//, "");
-          await options.render.beforeGoto(page, { route });
-          await page.goto(pageUrl, {
-            waitUntil: options.render.waitUntil,
-            timeout: options.render.timeoutMs
-          });
-          if (options.render.waitForSelector) {
-            await page.waitForSelector(options.render.waitForSelector, {
-              timeout: options.render.timeoutMs
-            });
-          }
-          if (options.render.postLoadDelayMs > 0) {
-            await new Promise(
-              (r) => setTimeout(r, options.render.postLoadDelayMs)
-            );
-          }
-          await options.render.beforeExtract(page, { route });
-          const html = await page.content();
-          const $ = cheerio.load(html);
-          let harvestedHrefs = [];
-          if ((_b2 = options.crawl) == null ? void 0 : _b2.enabled) {
-            harvestedHrefs = $("a[href]").map((_, a) => $(a).attr("href")).get();
-            log.debug(`  Found ${harvestedHrefs.length} links on ${route}:`, harvestedHrefs.slice(0, 15));
+      if (useStaticMode) {
+        log.debug("Using static mode - reading HTML files directly from dist/");
+        for (const rd of routeDefs) {
+          const route = rd.path;
+          if (isExcluded(route)) continue;
+          let htmlPath = routeToHtmlFsPath(distDir, route);
+          let htmlContent = null;
+          try {
+            htmlContent = await import_promises.default.readFile(htmlPath, "utf8");
+          } catch {
+            if (!route.endsWith("/") && route !== "/") {
+              const altPath = import_node_path.default.join(distDir, route.slice(1), "index.html");
+              try {
+                htmlContent = await import_promises.default.readFile(altPath, "utf8");
+                htmlPath = altPath;
+              } catch {
+                try {
+                  htmlContent = await import_promises.default.readFile(import_node_path.default.join(distDir, "index.html"), "utf8");
+                  htmlPath = import_node_path.default.join(distDir, "index.html");
+                  log.debug(`  Using SPA fallback index.html for ${route}`);
+                } catch {
+                  log.warn(`  \u26A0\uFE0F  No HTML found for ${route}`);
+                  continue;
+                }
+              }
+            }
           }
+          if (!htmlContent) continue;
+          const $ = cheerio.load(htmlContent);
           for (const sel of options.extract.removeSelectors || [])
             $(sel).remove();
           const mainSelectors = Array.isArray(options.extract.mainSelector) ? options.extract.mainSelector : [options.extract.mainSelector];
@@ -341,112 +307,240 @@ generated_at: ${(/* @__PURE__ */ new Date()).toISOString()}
 
 ` : "";
           await import_promises.default.writeFile(fsPath, frontmatter + markdownBody, "utf8");
-          const meta = routeDefs.find((r) => r.path === route);
           captured.push({
             route,
-            title: (meta == null ? void 0 : meta.title) || title,
-            section: (meta == null ? void 0 : meta.section) || "Pages",
-            optional: !!(meta == null ? void 0 : meta.optional),
-            notes: meta == null ? void 0 : meta.notes,
+            title: rd.title || title,
+            section: rd.section || "Pages",
+            optional: !!rd.optional,
+            notes: rd.notes,
             mdRelPath
           });
           log.info(`  \u2705 ${route} -> ${mdRelPath}`);
-          if ((_c2 = options.crawl) == null ? void 0 : _c2.enabled) {
-            for (const href of harvestedHrefs) {
-              const n = normalizeRoute(href, {
-                stripQuery: options.crawl.stripQuery
+        }
+      } else {
+        const previewServer = await (0, import_vite.preview)({
+          root: resolvedConfig.root,
+          base: resolvedConfig.base,
+          build: { outDir: distDir },
+          preview: { port: 0, open: false, host: "127.0.0.1" },
+          configFile: false,
+          plugins: [],
+          logLevel: "silent"
+        });
+        await new Promise((resolve, reject) => {
+          const server = previewServer.httpServer;
+          if (server.listening) {
+            resolve();
+          } else {
+            server.once("listening", resolve);
+            server.once("error", reject);
+            setTimeout(() => reject(new Error("Preview server failed to start")), 5e3);
+          }
+        });
+        const addr = previewServer.httpServer.address();
+        if (!addr || typeof addr === "string") {
+          await safeCloseHttpServer(previewServer.httpServer);
+          throw new Error("LLM Spider: could not determine preview server port");
+        }
+        const normalizedBase = basePath.endsWith("/") ? basePath : basePath + "/";
+        const baseUrl = `http://127.0.0.1:${addr.port}${normalizedBase}`;
+        log.debug("Preview server at:", baseUrl);
+        const pup = await loadPuppeteer();
+        const browser = await pup.launch(options.render.launchOptions);
+        const visited = /* @__PURE__ */ new Set();
+        const queue = [];
+        if ((_b = options.crawl) == null ? void 0 : _b.enabled) {
+          for (const seed of options.crawl.seeds || ["/"]) {
+            const nr = normalizeRoute(seed, {
+              stripQuery: options.crawl.stripQuery
+            });
+            if (nr) queue.push({ route: nr, depth: 0 });
+          }
+        } else {
+          for (const rd of routeDefs) queue.push({ route: rd.path, depth: 0 });
+        }
+        const maxDepth = ((_c = options.crawl) == null ? void 0 : _c.enabled) ? options.crawl.maxDepth : 0;
+        const maxPages = ((_d = options.crawl) == null ? void 0 : _d.enabled) ? options.crawl.maxPages : queue.length;
+        const concurrency = ((_e = options.crawl) == null ? void 0 : _e.enabled) ? options.crawl.concurrency : 3;
+        async function captureOne(route) {
+          var _a2, _b2, _c2;
+          if (visited.has(route)) return;
+          if (isExcluded(route)) return;
+          if (captured.length >= maxPages) return;
+          visited.add(route);
+          const page = await browser.newPage();
+          if ((_a2 = options.render.blockRequests) == null ? void 0 : _a2.length) {
+            await page.setRequestInterception(true);
+            page.on("request", (req) => {
+              const url = req.url();
+              const blocked = options.render.blockRequests.some(
+                (p) => p instanceof RegExp ? p.test(url) : url.includes(p)
+              );
+              if (blocked) req.abort();
+              else req.continue();
+            });
+          }
+          try {
+            const pageUrl = route === "/" ? baseUrl : baseUrl + route.replace(/^\//, "");
+            await options.render.beforeGoto(page, { route });
+            await page.goto(pageUrl, {
+              waitUntil: options.render.waitUntil,
+              timeout: options.render.timeoutMs
+            });
+            if (options.render.waitForSelector) {
+              await page.waitForSelector(options.render.waitForSelector, {
+                timeout: options.render.timeoutMs
               });
-              if (!n) continue;
-              let baseRelative = n;
-              if (normalizedBase !== "/" && baseRelative.startsWith(normalizedBase)) {
-                baseRelative = "/" + baseRelative.slice(normalizedBase.length);
-                baseRelative = baseRelative === "//" ? "/" : baseRelative.replace(/\/{2,}/g, "/");
+            }
+            if (options.render.postLoadDelayMs > 0) {
+              await new Promise(
+                (r) => setTimeout(r, options.render.postLoadDelayMs)
+              );
+            }
+            await options.render.beforeExtract(page, { route });
+            const html = await page.content();
+            const $ = cheerio.load(html);
+            let harvestedHrefs = [];
+            if ((_b2 = options.crawl) == null ? void 0 : _b2.enabled) {
+              harvestedHrefs = $("a[href]").map((_, a) => $(a).attr("href")).get();
+              log.debug(`  Found ${harvestedHrefs.length} links on ${route}:`, harvestedHrefs.slice(0, 15));
+            }
+            for (const sel of options.extract.removeSelectors || [])
+              $(sel).remove();
+            const mainSelectors = Array.isArray(options.extract.mainSelector) ? options.extract.mainSelector : [options.extract.mainSelector];
+            let mainHtml = null;
+            for (const sel of mainSelectors) {
+              if (!sel) continue;
+              const node = $(sel).first();
+              if (node && node.length) {
+                mainHtml = node.html();
+                break;
               }
-              if (!visited.has(baseRelative) && !isExcluded(baseRelative)) {
-                queue.push({ route: baseRelative, depth: -1 });
+            }
+            if (!mainHtml) {
+              const main = $("main").first();
+              mainHtml = main.length ? main.html() : $("body").html();
+            }
+            const title = ($("title").text() || "").trim() || route;
+            const markdownBody = turndown.turndown(mainHtml || "");
+            const mdRelPath = options.output.mode === "subdir" ? import_node_path.default.posix.join(options.output.subdir, routeToMdWebPath(route)) : routeToMdWebPath(route);
+            const fsPath = routeToMdFsPath(distDir, route);
+            await import_promises.default.mkdir(import_node_path.default.dirname(fsPath), { recursive: true });
+            const frontmatter = options.markdown.addFrontmatter ? `---
+source: ${route}
+title: ${title}
+generated_at: ${(/* @__PURE__ */ new Date()).toISOString()}
+---
+
+` : "";
+            await import_promises.default.writeFile(fsPath, frontmatter + markdownBody, "utf8");
+            const meta = routeDefs.find((r) => r.path === route);
+            captured.push({
+              route,
+              title: (meta == null ? void 0 : meta.title) || title,
+              section: (meta == null ? void 0 : meta.section) || "Pages",
+              optional: !!(meta == null ? void 0 : meta.optional),
+              notes: meta == null ? void 0 : meta.notes,
+              mdRelPath
+            });
+            log.info(`  \u2705 ${route} -> ${mdRelPath}`);
+            if ((_c2 = options.crawl) == null ? void 0 : _c2.enabled) {
+              for (const href of harvestedHrefs) {
+                const n = normalizeRoute(href, {
+                  stripQuery: options.crawl.stripQuery
+                });
+                if (!n) continue;
+                let baseRelative = n;
+                if (normalizedBase !== "/" && baseRelative.startsWith(normalizedBase)) {
+                  baseRelative = "/" + baseRelative.slice(normalizedBase.length);
+                  baseRelative = baseRelative === "//" ? "/" : baseRelative.replace(/\/{2,}/g, "/");
+                }
+                if (!visited.has(baseRelative) && !isExcluded(baseRelative)) {
+                  queue.push({ route: baseRelative, depth: -1 });
+                }
               }
             }
+          } catch (err) {
+            log.warn(`  \u26A0\uFE0F  failed ${route}: ${(err == null ? void 0 : err.message) || err}`);
+          } finally {
+            await page.close();
           }
-        } catch (err) {
-          log.warn(`  \u26A0\uFE0F  failed ${route}: ${(err == null ? void 0 : err.message) || err}`);
-        } finally {
-          await page.close();
         }
-      }
-      try {
-        while (queue.length && captured.length < maxPages) {
-          const batch = queue.splice(0, concurrency).map((item) => {
-            const depth = item.depth >= 0 ? item.depth : 1;
-            return { route: item.route, depth };
-          });
-          await Promise.all(
-            batch.map(async ({ route, depth }) => {
-              var _a2, _b2;
-              if (((_a2 = options.crawl) == null ? void 0 : _a2.enabled) && depth > maxDepth) return;
-              await captureOne(route);
-              if ((_b2 = options.crawl) == null ? void 0 : _b2.enabled) {
-                for (let i = 0; i < queue.length; i++) {
-                  if (queue[i].depth === -1) queue[i].depth = depth + 1;
+        try {
+          while (queue.length && captured.length < maxPages) {
+            const batch = queue.splice(0, concurrency).map((item) => {
+              const depth = item.depth >= 0 ? item.depth : 1;
+              return { route: item.route, depth };
+            });
+            await Promise.all(
+              batch.map(async ({ route, depth }) => {
+                var _a2, _b2;
+                if (((_a2 = options.crawl) == null ? void 0 : _a2.enabled) && depth > maxDepth) return;
+                await captureOne(route);
+                if ((_b2 = options.crawl) == null ? void 0 : _b2.enabled) {
+                  for (let i = 0; i < queue.length; i++) {
+                    if (queue[i].depth === -1) queue[i].depth = depth + 1;
+                  }
                 }
-              }
-            })
-          );
-        }
-        const llmsTitle = options.output.llmsTitle || ((_f = resolvedConfig == null ? void 0 : resolvedConfig.env) == null ? void 0 : _f.mode) || "Site";
-        const items = options.output.sort ? [...captured].sort((a, b) => a.route.localeCompare(b.route)) : captured;
-        const bySection = /* @__PURE__ */ new Map();
-        const optionalItems = [];
-        for (const item of items) {
-          if (item.optional) optionalItems.push(item);
-          else {
-            const s = item.section || "Pages";
-            bySection.set(s, [...bySection.get(s) || [], item]);
+              })
+            );
           }
+        } finally {
+          await browser.close();
+          await safeCloseHttpServer(previewServer.httpServer);
         }
-        let llms = `# ${llmsTitle}
+      }
+      const llmsTitle = options.output.llmsTitle || ((_f = resolvedConfig == null ? void 0 : resolvedConfig.env) == null ? void 0 : _f.mode) || "Site";
+      const items = options.output.sort ? [...captured].sort((a, b) => a.route.localeCompare(b.route)) : captured;
+      const bySection = /* @__PURE__ */ new Map();
+      const optionalItems = [];
+      for (const item of items) {
+        if (item.optional) optionalItems.push(item);
+        else {
+          const s = item.section || "Pages";
+          bySection.set(s, [...bySection.get(s) || [], item]);
+        }
+      }
+      let llms = `# ${llmsTitle}
 
 > ${options.output.llmsSummary}
 
 `;
-        for (const [section, sectionItems] of bySection.entries()) {
-          llms += `## ${section}
+      for (const [section, sectionItems] of bySection.entries()) {
+        llms += `## ${section}
 
 `;
-          for (const it of sectionItems) {
-            const link = makeLlmsLink(it.mdRelPath);
-            const label = it.title || it.route;
-            const notes = it.notes ? `: ${it.notes}` : "";
-            llms += `- [${label}](${link})${notes}
-`;
-          }
-          llms += `
+        for (const it of sectionItems) {
+          const link = makeLlmsLink(it.mdRelPath);
+          const label = it.title || it.route;
+          const notes = it.notes ? `: ${it.notes}` : "";
+          llms += `- [${label}](${link})${notes}
 `;
         }
-        if (optionalItems.length) {
-          llms += `## Optional
-
+        llms += `
 `;
-          for (const it of optionalItems) {
-            const link = makeLlmsLink(it.mdRelPath);
-            const label = it.title || it.route;
-            const notes = it.notes ? `: ${it.notes}` : "";
-            llms += `- [${label}](${link})${notes}
+      }
+      if (optionalItems.length) {
+        llms += `## Optional
+
 `;
-          }
-          llms += `
+        for (const it of optionalItems) {
+          const link = makeLlmsLink(it.mdRelPath);
+          const label = it.title || it.route;
+          const notes = it.notes ? `: ${it.notes}` : "";
+          llms += `- [${label}](${link})${notes}
 `;
         }
-        const llmsPath = import_node_path.default.join(distDir, options.output.llmsTxtFileName);
-        await import_promises.default.writeFile(llmsPath, llms, "utf8");
-        log.info(
-          `
+        llms += `
+`;
+      }
+      const llmsPath = import_node_path.default.join(distDir, options.output.llmsTxtFileName);
+      await import_promises.default.writeFile(llmsPath, llms, "utf8");
+      log.info(
+        `
 LLM Spider: wrote ${captured.length} markdown pages + ${options.output.llmsTxtFileName}
 `
-        );
-      } finally {
-        await browser.close();
-        await safeCloseHttpServer(previewServer.httpServer);
-      }
+      );
     }
   };
 }