@xyd-js/documan 0.0.0-build-6675456-20251014012658 → 0.0.0-build-1f6458c-20251015205119

package/dist/index.js

@@ -109,10 +109,19 @@ import { fileURLToPath } from "url";
 import { execSync as execSync2 } from "child_process";
 import crypto from "crypto";
 import { realpathSync } from "fs";
-import { createServer } from "vite";
+import * as fsPromises from "fs/promises";
+import { toMarkdown } from "mdast-util-to-markdown";
+import { u } from "unist-builder";
+import matter from "gray-matter";
+import {
+  createServer
+} from "vite";
 import { reactRouter } from "@react-router/dev/vite";
 import { IconSet } from "@iconify/tools";
-import { readSettings, pluginDocs } from "@xyd-js/plugin-docs";
+import {
+  readSettings,
+  pluginDocs
+} from "@xyd-js/plugin-docs";
 import { vitePlugins as xydContentVitePlugins } from "@xyd-js/content/vite";
 
 // ../xyd-plugin-supademo/package.json
@@ -480,6 +489,116 @@ async function installComponent(config, componentName) {
 // src/utils.ts
 var __filename = fileURLToPath(import.meta.url);
 var __dirname = path2.dirname(__filename);
+async function parseFrontmatterAndCreateAST(settings, content, filePath) {
+  if (settings?.ai?.llmsTxt === false) {
+    return null;
+  }
+  try {
+    const { data: fm } = matter(content) || { data: {} };
+    const title = fm?.title || "";
+    const description = fm?.description || "";
+    if (!title) {
+      return null;
+    }
+    const rawPath = filePath.replace(/^\/+/, "/");
+    const aiBaseUrl = typeof settings?.ai?.llmsTxt !== "string" ? settings?.ai?.llmsTxt?.baseUrl : "";
+    const isAbsolute = /^https?:\/\//i.test(rawPath);
+    const url = isAbsolute ? rawPath : aiBaseUrl ? aiBaseUrl.replace(/\/$/, "") + rawPath : rawPath;
+    const linkNode = u("link", { url }, [u("text", title)]);
+    const paragraphChildren = [linkNode];
+    if (description) {
+      paragraphChildren.push(u("text", `: ${description}`));
+    }
+    const paragraphNode = u("paragraph", paragraphChildren);
+    const listItemNode = u("listItem", [paragraphNode]);
+    return listItemNode;
+  } catch (error) {
+    console.warn(`Failed to parse frontmatter for ${filePath}:`, error);
+    return null;
+  }
+}
+function buildLLMsMarkdownAST(settings, rawRouteFiles, markdownItems) {
+  const aiLlms = settings?.ai?.llmsTxt;
+  if (aiLlms === false) {
+    return "";
+  }
+  const looksLikePath = isLLMsTxtPath(settings);
+  if (looksLikePath) {
+    const tryPaths = lookupPaths(aiLlms) || [];
+    for (const pth of tryPaths) {
+      const exists = fs.existsSync(pth);
+      if (!exists) {
+        continue;
+      }
+      const value = fs.readFileSync(pth, "utf8");
+      const normalized = value.split("\n").map((line) => line.replace(/^\s+(#{1,6}\s+)/, "$1")).join("\n").trim();
+      return normalized;
+    }
+  }
+  let h1Title = "";
+  const pathMappingIndex = globalThis.__xydPagePathMapping["index"];
+  if (pathMappingIndex) {
+    const indexContent = rawRouteFiles[normalizePath(pathMappingIndex)];
+    if (indexContent) {
+      try {
+        const { data } = matter(indexContent) || { data: {} };
+        if (data?.title && typeof data.title === "string") {
+          h1Title = data.title;
+        }
+      } catch {
+      }
+    }
+  }
+  let summaryText = "";
+  const extraBlocks = [];
+  if (aiLlms && typeof aiLlms === "object") {
+    const obj = aiLlms;
+    if (typeof obj.title === "string" && obj.title.trim()) {
+      h1Title = obj.title.trim();
+    }
+    if (typeof obj.summary === "string" && obj.summary.trim()) {
+      summaryText = obj.summary.trim();
+    }
+    if (obj.sections && typeof obj.sections === "object") {
+      for (const key of Object.keys(obj.sections)) {
+        const sec = obj.sections[key];
+        if (!sec || typeof sec !== "object") continue;
+        const sTitle = sec.title;
+        const sUrl = sec.url;
+        const sDescription = sec.description;
+        if (!sTitle) continue;
+        extraBlocks.push(u("heading", { depth: 2 }, [u("text", sTitle)]));
+        if (sUrl) {
+          const linkNode = u("link", { url: sUrl }, [u("text", sTitle)]);
+          const paraChildren = [linkNode];
+          if (sDescription) paraChildren.push(u("text", `: ${sDescription}`));
+          const listItem = u("listItem", [u("paragraph", paraChildren)]);
+          extraBlocks.push(
+            u("list", { ordered: false, spread: false }, [listItem])
+          );
+        } else if (sDescription) {
+          extraBlocks.push(u("paragraph", [u("text", sDescription)]));
+        }
+      }
+    }
+  }
+  const children = [];
+  children.push(u("heading", { depth: 1 }, [u("text", h1Title)]));
+  if (summaryText) {
+    children.push(u("paragraph", [u("text", summaryText)]));
+  }
+  children.push(u("heading", { depth: 2 }, [u("text", "Docs")]));
+  children.push(u("list", { ordered: false, spread: false }, markdownItems));
+  if (extraBlocks.length) {
+    children.push(...extraBlocks);
+  }
+  const root = u("root", children);
+  return toMarkdown(root, {
+    bullet: "-",
+    bulletOther: "*",
+    bulletOrdered: "."
+  });
+}
 var ANALYTICS_INTEGRATION_DEPENDENCIES = {
   livesession: {
     "@pluganalytics/provider-livesession": "0.0.0-pre.7"
@@ -491,6 +610,15 @@ var EXTERNAL_XYD_PLUGINS = {
   "@xyd-js/plugin-intercom": package_default3.version,
   "@xyd-js/plugin-livechat": package_default4.version
 };
+function isLLMsTxtPath(settings) {
+  const aiLlms = settings?.ai?.llmsTxt;
+  if (typeof aiLlms === "string" && aiLlms?.trim()) {
+    const value = aiLlms;
+    const looksLikePath = /\.(md|mdx|txt)$/i.test(value) || !value.includes("\n") && !value.startsWith("#") && !value.startsWith("- ");
+    return looksLikePath;
+  }
+  return false;
+}
 async function appInit(options) {
   const readPreloadSettings = await readSettings();
   if (!readPreloadSettings) {
@@ -562,9 +690,7 @@ async function appInit(options) {
       }
       const head = p.head;
       if (head?.length && preloadSettings?.theme?.head) {
-        preloadSettings.theme.head.push(
-          ...head
-        );
+        preloadSettings.theme.head.push(...head);
       }
     });
     globalThis.__xydUserUniformVitePlugins = userUniformVitePlugins;
@@ -594,12 +720,16 @@ async function appInit(options) {
   globalThis.__xydSettings = respPluginDocs.settings;
   globalThis.__xydPagePathMapping = respPluginDocs.pagePathMapping;
   globalThis.__xydHasIndexPage = respPluginDocs.hasIndexPage;
-  globalThis.__xydSettingsClone = JSON.parse(JSON.stringify(respPluginDocs.settings));
+  globalThis.__xydSettingsClone = JSON.parse(
+    JSON.stringify(respPluginDocs.settings)
+  );
+  globalThis.__xydRawRouteFiles = {};
   if (respPluginDocs.settings.integrations?.diagrams) {
     if (!componentExists("diagrams")) {
       await componentsInstall("diagrams");
     }
   }
+  await pluginLLMMarkdown(respPluginDocs);
   return {
     respPluginDocs,
     resolvedPlugins
@@ -660,9 +790,7 @@ function virtualProvidersPlugin(settings) {
         const imports = providers.map(
           (provider) => `import { default as ${provider}Provider } from '@pluganalytics/provider-${provider}'`
         ).join("\n");
-        const cases = providers.map(
-          (provider) => `case '${provider}': return ${provider}Provider`
-        ).join("\n");
+        const cases = providers.map((provider) => `case '${provider}': return ${provider}Provider`).join("\n");
         return `
                     ${imports}
 
@@ -696,10 +824,66 @@ async function commonVitePlugins(respPluginDocs, resolvedPlugins) {
     ...userVitePlugins
   ];
 }
+async function pluginLLMMarkdown(respPluginDocs) {
+  const paths = Object.keys(globalThis.__xydPagePathMapping || {});
+  const rawRouteFiles = {};
+  const llmsItems = [];
+  for (const key of paths) {
+    const mappingPath = globalThis.__xydPagePathMapping[key];
+    const savePath = path2.join(process.cwd(), mappingPath);
+    const savePathExt = path2.extname(savePath);
+    if (savePathExt === ".md" || savePathExt === ".mdx") {
+      try {
+        await fsPromises.stat(savePath);
+      } catch (e) {
+        continue;
+      }
+      const content = await fsPromises.readFile(savePath, "utf8");
+      const formattedKey = normalizePath(key);
+      const keyWithExt = formattedKey.endsWith(savePathExt) ? formattedKey : `${formattedKey}${savePathExt}`;
+      rawRouteFiles[keyWithExt] = content;
+      const listItem = await parseFrontmatterAndCreateAST(
+        respPluginDocs.settings,
+        content,
+        keyWithExt
+      );
+      if (listItem) {
+        llmsItems.push(listItem);
+      }
+    }
+  }
+  if (respPluginDocs.settings?.ai?.llmsTxt !== false) {
+    let foundUserLLMsTxt = false;
+    for (const pth of lookupPaths("llms.txt")) {
+      const exists = fs.existsSync(pth);
+      if (!exists) {
+        continue;
+      }
+      const llmContent = fs.readFileSync(pth, "utf8");
+      rawRouteFiles["/llms.txt"] = llmContent;
+      foundUserLLMsTxt = true;
+      break;
+    }
+    if (!foundUserLLMsTxt && (llmsItems.length > 0 || respPluginDocs.settings?.ai?.llmsTxt)) {
+      const llmsContent = buildLLMsMarkdownAST(
+        respPluginDocs.settings,
+        rawRouteFiles,
+        llmsItems
+      );
+      rawRouteFiles["/llms.txt"] = llmsContent;
+    }
+  }
+  globalThis.__xydRawRouteFiles = {
+    ...globalThis.__xydRawRouteFiles,
+    ...rawRouteFiles
+  };
+}
+function normalizePath(p) {
+  const formattedKey = p.startsWith("/") ? p : `/${p}`;
+  return formattedKey;
+}
 function commonPostInstallVitePlugins(respPluginDocs, resolvedPlugins) {
-  return [
-    vitePluginThemePresets(respPluginDocs.settings)
-  ];
+  return [vitePluginThemePresets(respPluginDocs.settings)];
 }
 async function vitePluginThemePresets(settings) {
   const themeName = settings.theme?.name;
@@ -709,9 +893,15 @@ async function vitePluginThemePresets(settings) {
   const __dirname2 = path2.dirname(__filename2);
   let themeRoot = "";
   if (process.env.XYD_CLI) {
-    themeRoot = path2.join(getHostPath(), `node_modules/@xyd-js/theme-${themeName}/dist`);
+    themeRoot = path2.join(
+      getHostPath(),
+      `node_modules/@xyd-js/theme-${themeName}/dist`
+    );
   } else {
-    themeRoot = path2.join(path2.resolve(__dirname2, "../../"), `xyd-theme-${themeName}/dist`);
+    themeRoot = path2.join(
+      path2.resolve(__dirname2, "../../"),
+      `xyd-theme-${themeName}/dist`
+    );
   }
   const presetsDir = path2.join(themeRoot, "presets");
   let cssFiles = [];
@@ -795,7 +985,9 @@ function pluginIconSet(settings) {
       const iconSet = new IconSet(iconsData);
       return { icons: iconsData, iconSet };
     } catch (error) {
-      throw new Error(`Failed to load icon set from any source (file or CDN): ${name}`);
+      throw new Error(
+        `Failed to load icon set from any source (file or CDN): ${name}`
+      );
     }
   }
   async function processIconSet(iconSet, icons, noPrefix) {
@@ -864,10 +1056,7 @@ function pluginIconSet(settings) {
   };
 }
 function getXydFolderPath() {
-  return path2.join(
-    process.cwd(),
-    XYD_FOLDER_PATH
-  );
+  return path2.join(process.cwd(), XYD_FOLDER_PATH);
 }
 function getCLIRoot() {
   const cliPath = realpathSync(process.argv[1]);
@@ -901,10 +1090,7 @@ function getPublicPath() {
   return path2.join(process.cwd(), "public");
 }
 function getBuildPath() {
-  return path2.join(
-    process.cwd(),
-    BUILD_FOLDER_PATH
-  );
+  return path2.join(process.cwd(), BUILD_FOLDER_PATH);
 }
 function getDocsPluginBasePath() {
   return path2.join(getHostPath(), "./plugins/xyd-plugin-docs");
@@ -925,14 +1111,20 @@ async function loadPlugins(settings, options) {
       pluginName = plugin[0];
       pluginArgs = plugin.slice(1);
     } else {
-      console.error(`Currently only string and array plugins are supported, got: ${plugin}`);
+      console.error(
+        `Currently only string and array plugins are supported, got: ${plugin}`
+      );
       return [];
     }
     let mod;
     try {
       mod = await import(pluginName);
     } catch (e) {
-      pluginName = path2.join(process.cwd(), ".xyd/host/node_modules", pluginName);
+      pluginName = path2.join(
+        process.cwd(),
+        ".xyd/host/node_modules",
+        pluginName
+      );
       const pluginPreview = await createServer({
         optimizeDeps: {
           include: []
@@ -992,22 +1184,13 @@ function integrationsToPlugins(integrations) {
     plugins.push(["@xyd-js/plugin-supademo", integrations[".apps"].supademo]);
   }
   if (integrations?.support?.chatwoot) {
-    plugins.push([
-      "@xyd-js/plugin-chatwoot",
-      integrations.support.chatwoot
-    ]);
+    plugins.push(["@xyd-js/plugin-chatwoot", integrations.support.chatwoot]);
   }
   if (integrations?.support?.intercom) {
-    plugins.push([
-      "@xyd-js/plugin-intercom",
-      integrations.support.intercom
-    ]);
+    plugins.push(["@xyd-js/plugin-intercom", integrations.support.intercom]);
   }
   if (integrations?.support?.livechat) {
-    plugins.push([
-      "@xyd-js/plugin-livechat",
-      integrations.support.livechat
-    ]);
+    plugins.push(["@xyd-js/plugin-livechat", integrations.support.livechat]);
   }
   return plugins;
 }
@@ -1028,7 +1211,10 @@ async function preWorkspaceSetup(options = {}) {
     pluginDocsPath = path2.resolve(__dirname, "../../plugin-docs");
   }
   const pagesSourcePath = path2.join(pluginDocsPath, "src/pages");
-  const pagesTargetPath = path2.join(hostPath, "plugins/xyd-plugin-docs/src/pages");
+  const pagesTargetPath = path2.join(
+    hostPath,
+    "plugins/xyd-plugin-docs/src/pages"
+  );
   if (fs.existsSync(pagesSourcePath)) {
     await copyHostTemplate(pagesSourcePath, pagesTargetPath);
   } else {
@@ -1037,7 +1223,16 @@ async function preWorkspaceSetup(options = {}) {
 }
 function calculateFolderChecksum(folderPath) {
   const hash = crypto.createHash("sha256");
-  const ignorePatterns = [...getGitignorePatterns(folderPath), ".xydchecksum", "node_modules", "dist", ".react-router", "package-lock.json", "pnpm-lock.yaml", "cliComponents.json"];
+  const ignorePatterns = [
+    ...getGitignorePatterns(folderPath),
+    ".xydchecksum",
+    "node_modules",
+    "dist",
+    ".react-router",
+    "package-lock.json",
+    "pnpm-lock.yaml",
+    "cliComponents.json"
+  ];
   function processFile(filePath) {
     const relativePath = path2.relative(folderPath, filePath);
     const content = fs.readFileSync(filePath);
@@ -1104,7 +1299,9 @@ async function copyHostTemplate(sourcePath, targetPath) {
       fs.copyFileSync(sourceEntry, targetEntry);
       if (entry.name === "package.json" && process.env.XYD_DEV_MODE) {
         const packageJsonPath = targetEntry;
-        const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf-8"));
+        const packageJson = JSON.parse(
+          fs.readFileSync(packageJsonPath, "utf-8")
+        );
         packageJson.name = "xyd-host-dev";
         fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2));
       }
@@ -1168,7 +1365,9 @@ async function hostPackageJson() {
 }
 async function setupIntegationDependencies(settings) {
   const dependencies = {};
-  for (const [key, value] of Object.entries(ANALYTICS_INTEGRATION_DEPENDENCIES)) {
+  for (const [key, value] of Object.entries(
+    ANALYTICS_INTEGRATION_DEPENDENCIES
+  )) {
     const analytics = settings.integrations?.analytics?.[key];
     if (analytics) {
       for (const [depName, depVersion] of Object.entries(value)) {
@@ -1201,11 +1400,15 @@ async function setupPluginDependencies(settings, install2 = false) {
       const cwdPackageJsonPath = path2.join(process.cwd(), "package.json");
       let userDeps = {};
       if (fs.existsSync(cwdPackageJsonPath)) {
-        const cwdPackageJson = JSON.parse(fs.readFileSync(cwdPackageJsonPath, "utf-8"));
+        const cwdPackageJson = JSON.parse(
+          fs.readFileSync(cwdPackageJsonPath, "utf-8")
+        );
         userDeps = cwdPackageJson.dependencies || {};
       }
       if (fs.existsSync(hostPackageJsonPath)) {
-        const hostPackageJson2 = JSON.parse(fs.readFileSync(hostPackageJsonPath, "utf-8"));
+        const hostPackageJson2 = JSON.parse(
+          fs.readFileSync(hostPackageJsonPath, "utf-8")
+        );
         const deps = hostPackageJson2.dependencies || {};
         const matchingUserDep = Object.entries(userDeps).find(([depName]) => {
           return depName === pluginName;
@@ -1281,7 +1484,9 @@ function pmInstall() {
         return bunInstall();
       }
       default: {
-        console.warn(`Unknown package manager: ${process.env.XYD_NODE_PM}, falling back to npm`);
+        console.warn(
+          `Unknown package manager: ${process.env.XYD_NODE_PM}, falling back to npm`
+        );
         return npmInstall();
       }
     }
@@ -1363,6 +1568,24 @@ function getStoredChecksum() {
     return null;
   }
 }
+function lookupPaths(value) {
+  const basePath = globalThis.__xydBasePath;
+  const tryPaths = [];
+  if (path2.isAbsolute(value)) {
+    tryPaths.push(value);
+  } else {
+    tryPaths.push(
+      path2.join(process.cwd(), value),
+      path2.join(process.cwd(), "public", value),
+      path2.join(getAppRoot(), value),
+      path2.join(getHostPath(), value)
+    );
+    if (typeof basePath === "string" && basePath) {
+      tryPaths.push(path2.join(basePath, value));
+    }
+  }
+  return tryPaths;
+}
 function storeChecksum(checksum) {
   const checksumPath = path2.join(getHostPath(), ".xydchecksum");
   try {
@@ -1832,6 +2055,7 @@ async function dev(options) {
     if (isContentFile && !renameContentFile) {
       console.log("\u{1F504} Content file changed, refresh...");
       invalidateSettings(server);
+      pluginLLMMarkdown(respPluginDocs);
       await touchLayoutPage();
       console.log("\u2714 xyd content file changed\n");
       return;