@marlinjai/clearify 1.20.0 → 2.1.0

package/dist/config-schema.json

@@ -116,9 +116,6 @@
               "description": {
                 "type": "string"
               },
-              "href": {
-                "type": "string"
-              },
               "repo": {
                 "type": "string"
               },
@@ -144,64 +141,165 @@
               "group": {
                 "type": "string"
               },
-              "mode": {
-                "type": "string",
-                "enum": [
-                  "link",
-                  "embed",
-                  "inject"
-                ]
-              },
-              "git": {
-                "type": "object",
-                "properties": {
-                  "repo": {
-                    "type": "string"
+              "source": {
+                "oneOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "kind": {
+                        "type": "string",
+                        "const": "none"
+                      }
+                    },
+                    "required": [
+                      "kind"
+                    ],
+                    "additionalProperties": false
                   },
-                  "ref": {
-                    "type": "string"
+                  {
+                    "type": "object",
+                    "properties": {
+                      "kind": {
+                        "type": "string",
+                        "const": "git"
+                      },
+                      "repo": {
+                        "type": "string"
+                      },
+                      "ref": {
+                        "type": "string"
+                      },
+                      "path": {
+                        "type": "string"
+                      },
+                      "sparse": {
+                        "type": "boolean"
+                      }
+                    },
+                    "required": [
+                      "kind",
+                      "repo"
+                    ],
+                    "additionalProperties": false
                   },
-                  "path": {
-                    "type": "string"
+                  {
+                    "type": "object",
+                    "properties": {
+                      "kind": {
+                        "type": "string",
+                        "const": "url"
+                      },
+                      "url": {
+                        "type": "string"
+                      }
+                    },
+                    "required": [
+                      "kind",
+                      "url"
+                    ],
+                    "additionalProperties": false
                   },
-                  "sparse": {
-                    "type": "boolean"
+                  {
+                    "type": "object",
+                    "properties": {
+                      "kind": {
+                        "type": "string",
+                        "const": "inline"
+                      },
+                      "markdown": {
+                        "type": "string"
+                      }
+                    },
+                    "required": [
+                      "kind",
+                      "markdown"
+                    ],
+                    "additionalProperties": false
                   }
-                },
-                "required": [
-                  "repo"
-                ],
-                "additionalProperties": false
+                ]
               },
-              "embedSections": {
-                "anyOf": [
+              "placement": {
+                "oneOf": [
                   {
-                    "type": "string",
-                    "const": "all"
+                    "type": "object",
+                    "properties": {
+                      "kind": {
+                        "type": "string",
+                        "const": "card"
+                      },
+                      "href": {
+                        "type": "string"
+                      }
+                    },
+                    "required": [
+                      "kind",
+                      "href"
+                    ],
+                    "additionalProperties": false
                   },
                   {
-                    "type": "string",
-                    "const": "public"
+                    "type": "object",
+                    "properties": {
+                      "kind": {
+                        "type": "string",
+                        "const": "tab"
+                      },
+                      "sections": {
+                        "anyOf": [
+                          {
+                            "type": "string",
+                            "const": "all"
+                          },
+                          {
+                            "type": "string",
+                            "const": "public"
+                          },
+                          {
+                            "type": "array",
+                            "items": {
+                              "type": "string"
+                            }
+                          }
+                        ]
+                      }
+                    },
+                    "required": [
+                      "kind"
+                    ],
+                    "additionalProperties": false
                   },
                   {
-                    "type": "array",
-                    "items": {
-                      "type": "string"
-                    }
+                    "type": "object",
+                    "properties": {
+                      "kind": {
+                        "type": "string",
+                        "const": "nested"
+                      },
+                      "into": {
+                        "type": "string"
+                      },
+                      "docsPath": {
+                        "type": "string"
+                      },
+                      "group": {
+                        "type": "string"
+                      }
+                    },
+                    "required": [
+                      "kind",
+                      "into"
+                    ],
+                    "additionalProperties": false
                   }
                 ]
-              },
-              "injectInto": {
-                "type": "string"
-              },
-              "docsPath": {
-                "type": "string"
               }
             },
             "required": [
               "name",
               "description",
-              "status"
+              "status",
+              "source",
+              "placement"
             ],
             "additionalProperties": false
           }

package/dist/node/{chunk-JZZ2BNIL.js → chunk-GPBEDIOY.js}

@@ -282,7 +282,7 @@ ${hasInternal ? `
 `);
   const secretMode = options.secretMode ?? "prompt";
   if (options.hub) {
-    const { registerWithHub } = await import("./hub-register-7NF5BP3E.js");
+    const { registerWithHub } = await import("./hub-register-QB73GUF2.js");
     await registerWithHub({
       projectName: detectProjectName(),
       hubToken: options.hubToken,
@@ -298,7 +298,7 @@ ${hasInternal ? `
     });
     rl.close();
     if (answer.toLowerCase() === "y") {
-      const { registerWithHub } = await import("./hub-register-7NF5BP3E.js");
+      const { registerWithHub } = await import("./hub-register-QB73GUF2.js");
       await registerWithHub({
         projectName: detectProjectName(),
         hubToken: options.hubToken,

package/dist/node/{chunk-HZRIVY25.js → chunk-GZE6QLJT.js}

@@ -2,7 +2,7 @@ import {
   loadUserConfig,
   resolveConfig,
   resolveSections
-} from "./chunk-2WML2D77.js";
+} from "./chunk-ZWF6VCSQ.js";
 import {
   scanDocs
 } from "./chunk-CTL4YL7C.js";

package/dist/node/{chunk-XR4WG237.js → chunk-PZA4UAFG.js}

@@ -6,14 +6,18 @@ import {
 import {
   mermaidContentHash
 } from "./chunk-SCZZB7OE.js";
+import {
+  generateLlmsFullTxt,
+  generateLlmsTxt
+} from "./chunk-TVDWHKQF.js";
 import {
   clearifyPlugin
-} from "./chunk-GWYMXKDL.js";
+} from "./chunk-Q5FUAK6J.js";
 import {
   loadUserConfig,
   resolveConfig,
   resolveSections
-} from "./chunk-2WML2D77.js";
+} from "./chunk-ZWF6VCSQ.js";
 import {
   buildSectionData
 } from "./chunk-CTL4YL7C.js";
@@ -72,6 +76,29 @@ function generateSitemap(routes, siteUrl) {
 ${urls.join("\n")}
 </urlset>`;
 }
+function formatBytes(bytes) {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
+}
+function collectLlmsDocs(sections) {
+  const out = [];
+  for (const sd of sections) {
+    for (const doc of sd.docs) {
+      const title = doc.frontmatter.title ?? "Untitled";
+      const optional = (doc.frontmatter.order ?? 0) >= 9e3;
+      const markdown = existsSync(doc.filePath) ? readFileSync(doc.filePath, "utf-8") : "";
+      out.push({
+        title,
+        routePath: doc.routePath,
+        sectionLabel: sd.section.label,
+        markdown,
+        optional
+      });
+    }
+  }
+  return out;
+}
 function generateRobotsTxt(siteUrl) {
   const lines = ["User-agent: *", "Allow: /"];
   if (siteUrl) {
@@ -312,6 +339,19 @@ async function buildSite() {
   const sitemap = generateSitemap(sitemapRoutes, config.siteUrl);
   writeFileSync(resolve(outDir, "sitemap.xml"), sitemap);
   console.log("  Generated sitemap.xml");
+  if (config.generateLlmsTxt !== false) {
+    const llmsDocs = collectLlmsDocs(sectionDataList);
+    const llmsTxt = generateLlmsTxt(config, llmsDocs);
+    const llmsFullTxt = generateLlmsFullTxt(config, llmsDocs);
+    const llmsTxtPath = resolve(outDir, "llms.txt");
+    const llmsFullTxtPath = resolve(outDir, "llms-full.txt");
+    writeFileSync(llmsTxtPath, llmsTxt);
+    writeFileSync(llmsFullTxtPath, llmsFullTxt);
+    const llmsTxtSize = Buffer.byteLength(llmsTxt, "utf-8");
+    const llmsFullSize = Buffer.byteLength(llmsFullTxt, "utf-8");
+    console.log(`  Generated ${llmsTxtPath} (${formatBytes(llmsTxtSize)})`);
+    console.log(`  Generated ${llmsFullTxtPath} (${formatBytes(llmsFullSize)})`);
+  }
   rmSync(ssrOutDir, { recursive: true, force: true });
   console.log("  Cleaned up SSR build artifacts.");
   if (mermaidStrategy === "build") {

package/dist/node/{chunk-GWYMXKDL.js → chunk-Q5FUAK6J.js}

@@ -10,7 +10,7 @@ import {
   resolveSections,
   scanHubProjects,
   writeDataConfig
-} from "./chunk-2WML2D77.js";
+} from "./chunk-ZWF6VCSQ.js";
 import {
   buildSectionData
 } from "./chunk-CTL4YL7C.js";

package/dist/node/chunk-TVDWHKQF.js

@@ -0,0 +1,134 @@
+// src/core/llms-txt.ts
+import { readFileSync } from "fs";
+import matter from "gray-matter";
+function normalizeBase(siteUrl) {
+  if (!siteUrl) return "";
+  return siteUrl.replace(/\/$/, "");
+}
+function toAbsoluteUrl(base, routePath) {
+  if (!base) return routePath;
+  if (!routePath.startsWith("/")) return `${base}/${routePath}`;
+  return `${base}${routePath}`;
+}
+function stripFrontmatterAndH1(markdown) {
+  const { content } = matter(markdown);
+  return content.replace(/^\s*#\s+[^\n]*\n+/, "").trimStart();
+}
+function firstParagraphSummary(markdown) {
+  const parsed = matter(markdown);
+  const fromFrontmatter = typeof parsed.data.summary === "string" && parsed.data.summary.trim() || typeof parsed.data.description === "string" && parsed.data.description.trim() || "";
+  const body = parsed.content;
+  const lines = body.split(/\r?\n/);
+  let inFence = false;
+  const paragraph = [];
+  for (const rawLine of lines) {
+    const line = rawLine.trimEnd();
+    if (/^```/.test(line)) {
+      inFence = !inFence;
+      if (paragraph.length > 0) break;
+      continue;
+    }
+    if (inFence) continue;
+    if (!line.trim()) {
+      if (paragraph.length > 0) break;
+      continue;
+    }
+    if (/^(#{1,6}\s|>\s|[-*+]\s|\d+\.\s|\|.*\|\s*$|<)/.test(line)) {
+      if (paragraph.length > 0) break;
+      continue;
+    }
+    paragraph.push(line.trim());
+  }
+  const text = paragraph.join(" ").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/`([^`]+)`/g, "$1").replace(/([*_]{1,3})([^*_]+)\1/g, "$2").replace(/\s+/g, " ").trim();
+  return text || fromFrontmatter;
+}
+function uniqueOrdered(items) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const item of items) {
+    if (seen.has(item)) continue;
+    seen.add(item);
+    out.push(item);
+  }
+  return out;
+}
+function buildHeader(config) {
+  const name = config.name.trim();
+  const tagline = config.hubProject?.description && config.hubProject.description.trim() || `Documentation for ${name}.`;
+  return `# ${name}
+
+> ${tagline}
+
+`;
+}
+function generateLlmsTxt(config, docs) {
+  const base = normalizeBase(config.siteUrl);
+  let out = buildHeader(config);
+  const mainDocs = docs.filter((d) => !d.optional);
+  const optionalDocs = docs.filter((d) => d.optional);
+  const sectionLabels = uniqueOrdered(mainDocs.map((d) => d.sectionLabel));
+  for (const label of sectionLabels) {
+    const sectionDocs = mainDocs.filter((d) => d.sectionLabel === label);
+    if (sectionDocs.length === 0) continue;
+    out += `## ${label}
+
+`;
+    for (const doc of sectionDocs) {
+      const url = toAbsoluteUrl(base, doc.routePath);
+      const summary = firstParagraphSummary(doc.markdown);
+      if (summary) {
+        out += `- [${doc.title}](${url}): ${summary}
+`;
+      } else {
+        out += `- [${doc.title}](${url})
+`;
+      }
+    }
+    out += "\n";
+  }
+  if (optionalDocs.length > 0) {
+    out += `## Optional
+
+`;
+    for (const doc of optionalDocs) {
+      const url = toAbsoluteUrl(base, doc.routePath);
+      const summary = firstParagraphSummary(doc.markdown);
+      if (summary) {
+        out += `- [${doc.title}](${url}): ${summary}
+`;
+      } else {
+        out += `- [${doc.title}](${url})
+`;
+      }
+    }
+    out += "\n";
+  }
+  return out.trimEnd() + "\n";
+}
+function generateLlmsFullTxt(config, docs) {
+  const base = normalizeBase(config.siteUrl);
+  let out = buildHeader(config);
+  for (const doc of docs) {
+    const url = toAbsoluteUrl(base, doc.routePath);
+    const body = stripFrontmatterAndH1(doc.markdown);
+    out += `## ${doc.title}
+
+`;
+    out += `Source: ${url}
+
+`;
+    if (body) {
+      out += body.trimEnd() + "\n\n";
+    }
+  }
+  return out.trimEnd() + "\n";
+}
+function readMarkdownFile(filePath) {
+  return readFileSync(filePath, "utf-8");
+}
+
+export {
+  generateLlmsTxt,
+  generateLlmsFullTxt,
+  readMarkdownFile
+};

package/dist/node/{chunk-2WML2D77.js → chunk-ZWF6VCSQ.js}

@@ -42,28 +42,44 @@ var OpenAPIConfigSchema = z.object({
   basePath: z.string().default("/api"),
   generatePages: z.boolean().default(true)
 }).optional();
+var HubProjectSourceSchema = z.discriminatedUnion("kind", [
+  z.object({ kind: z.literal("none") }),
+  z.object({
+    kind: z.literal("git"),
+    repo: z.string(),
+    ref: z.string().optional(),
+    path: z.string().optional(),
+    sparse: z.boolean().optional()
+  }),
+  z.object({ kind: z.literal("url"), url: z.string() }),
+  z.object({ kind: z.literal("inline"), markdown: z.string() })
+]);
+var HubProjectPlacementSchema = z.discriminatedUnion("kind", [
+  z.object({ kind: z.literal("card"), href: z.string() }),
+  z.object({
+    kind: z.literal("tab"),
+    sections: z.union([z.literal("all"), z.literal("public"), z.array(z.string())]).optional()
+  }),
+  z.object({
+    kind: z.literal("nested"),
+    into: z.string(),
+    docsPath: z.string().optional(),
+    group: z.string().optional()
+  })
+]);
 var HubProjectSchema = z.object({
   name: z.string(),
   description: z.string(),
-  href: z.string().optional(),
   repo: z.string().optional(),
   status: z.enum(["active", "beta", "planned", "deprecated"]).default("active"),
   icon: z.string().optional(),
   tags: z.array(z.string()).optional(),
   group: z.string().optional(),
-  mode: z.enum(["link", "embed", "inject"]).optional(),
-  git: RemoteGitSourceSchema.optional(),
-  embedSections: z.union([
-    z.literal("all"),
-    z.literal("public"),
-    z.array(z.string())
-  ]).optional(),
-  injectInto: z.string().optional(),
-  docsPath: z.string().optional()
+  source: HubProjectSourceSchema,
+  placement: HubProjectPlacementSchema
 });
 var HubProjectPartialSchema = z.object({
   description: z.string(),
-  href: z.string().optional(),
   repo: z.string().optional(),
   status: z.enum(["active", "beta", "planned", "deprecated"]).default("active"),
   icon: z.string().optional(),
@@ -101,7 +117,8 @@ var ClearifyConfigSchema = z.object({
   hubProject: HubProjectPartialSchema,
   customCss: z.string().optional(),
   headTags: z.array(z.string()).optional(),
-  includeReadme: z.boolean().default(false)
+  includeReadme: z.boolean().default(false),
+  generateLlmsTxt: z.boolean().default(true)
 });
 var ClearifyDataSchema = z.object({
   name: z.string().optional(),
@@ -249,6 +266,11 @@ function resolveConfig(userConfig = {}, root) {
   if (!userConfig.name && root) {
     parsed.name = detectProjectName(root);
   }
+  if (parsed.generateLlmsTxt && !parsed.siteUrl) {
+    throw new Error(
+      "clearify config: `generateLlmsTxt` is enabled (default) but `siteUrl` is not set. Set `siteUrl` to the production URL of your docs site, or set `generateLlmsTxt: false` to disable."
+    );
+  }
   return parsed;
 }
 function slugify(str) {
@@ -335,8 +357,14 @@ function buildStagingOverlay(stagingDir, originalDir, injections) {
     symlinkSync(sourcePath, destPath);
   }
 }
+function isReservedSource(source) {
+  return source.kind === "url" || source.kind === "inline";
+}
 async function scanHubProjects(config, root) {
-  if (!config.hub?.scan && !config.hub?.projects?.some((p) => p.mode === "embed" || p.mode === "inject")) {
+  const hasGitTabOrNested = config.hub?.projects?.some(
+    (p) => p.source.kind === "git" && (p.placement.kind === "tab" || p.placement.kind === "nested")
+  ) ?? false;
+  if (!config.hub?.scan && !hasGitTabOrNested) {
     return config;
   }
   const scannedProjects = [];
@@ -350,15 +378,17 @@ async function scanHubProjects(config, root) {
         if (!childConfig.hubProject) continue;
         const name = childConfig.name ?? basename(dirname(configPath));
         const hp = childConfig.hubProject;
+        const href = childConfig.siteUrl;
         scannedProjects.push({
           name,
           description: hp.description,
-          href: hp.href ?? childConfig.siteUrl,
           repo: hp.repo,
           status: hp.status,
           icon: hp.icon,
           tags: hp.tags,
-          group: hp.group
+          group: hp.group,
+          source: { kind: "none" },
+          placement: { kind: "card", href: href ?? "#" }
         });
       } catch {
       }
@@ -369,18 +399,26 @@ async function scanHubProjects(config, root) {
     ...scannedProjects.filter((p) => !manualNames.has(p.name)),
     ...config.hub?.projects ?? []
   ];
+  for (const project of merged) {
+    if (isReservedSource(project.source)) {
+      throw new Error(
+        `Hub project "${project.name}": source.kind "${project.source.kind}" is reserved and not implemented. Use source.kind "none" or "git".`
+      );
+    }
+  }
   const embedSections = [];
   for (const project of merged) {
-    if (project.mode !== "embed" || !project.git) continue;
+    if (project.source.kind !== "git" || project.placement.kind !== "tab") continue;
     const cacheDir = config.hub?.cacheDir ? resolve(root, config.hub.cacheDir) : resolve(root, "node_modules/.cache/clearify-remote");
     try {
       const { resolveRemoteSource } = await import("./core/remote.js");
-      const { localPath } = await resolveRemoteSource(project.git, cacheDir);
+      const { kind: _kind, ...gitSource } = project.source;
+      const { localPath } = await resolveRemoteSource(gitSource, cacheDir);
       const remoteConfig = await loadUserConfig(localPath);
       const remoteSections = remoteConfig.sections ?? [
         { label: remoteConfig.name ?? project.name, docsDir: remoteConfig.docsDir ?? "./docs" }
       ];
-      const filter = project.embedSections ?? "public";
+      const filter = project.placement.sections ?? "public";
       const filtered = remoteSections.filter((s) => {
         if (filter === "all") return true;
         if (filter === "public") return !s.draft;
@@ -400,46 +438,44 @@ async function scanHubProjects(config, root) {
           exclude: s.exclude
         });
       }
-      console.log(`  Hub embed "${project.name}": ${filtered.length} sections pulled`);
+      console.log(`  Hub tab "${project.name}": ${filtered.length} sections pulled`);
     } catch (err) {
       console.warn(
-        `  \u26A0 Hub embed "${project.name}" failed:`,
+        `  Hub tab "${project.name}" failed:`,
         err instanceof Error ? err.message : err
       );
     }
   }
   const injectionsBySection = /* @__PURE__ */ new Map();
   for (const project of merged) {
-    if (project.mode !== "inject" || !project.git) continue;
+    if (project.source.kind !== "git" || project.placement.kind !== "nested") continue;
     const cacheDir = config.hub?.cacheDir ? resolve(root, config.hub.cacheDir) : resolve(root, "node_modules/.cache/clearify-remote");
     try {
       const { resolveRemoteSource } = await import("./core/remote.js");
-      const { localPath, cached } = await resolveRemoteSource(project.git, cacheDir);
-      const docsPath = project.docsPath ?? "docs";
+      const { kind: _kind, ...gitSource } = project.source;
+      const { localPath, cached } = await resolveRemoteSource(gitSource, cacheDir);
+      const docsPath = project.placement.docsPath ?? "docs";
       const absoluteDocsPath = resolve(localPath, docsPath);
       if (!existsSync(absoluteDocsPath)) {
         console.warn(
-          `  \u26A0 Hub inject "${project.name}": docs path "${docsPath}" not found in cloned repo.`
+          `  Hub nested "${project.name}": docs path "${docsPath}" not found in cloned repo.`
         );
         continue;
       }
-      const targetLabel = project.injectInto;
-      if (!targetLabel) {
-        console.warn(`  \u26A0 Hub inject "${project.name}": missing injectInto field.`);
-        continue;
-      }
+      const targetLabel = project.placement.into;
       if (!injectionsBySection.has(targetLabel)) {
         injectionsBySection.set(targetLabel, /* @__PURE__ */ new Map());
       }
       const slugifiedName = slugify(project.name);
-      const relPath = project.group ? `projects/${project.group}/${slugifiedName}` : `projects/${slugifiedName}`;
+      const groupSeg = project.placement.group;
+      const relPath = groupSeg ? `projects/${groupSeg}/${slugifiedName}` : `projects/${slugifiedName}`;
       injectionsBySection.get(targetLabel).set(relPath, absoluteDocsPath);
       console.log(
-        `  Hub inject "${project.name}": ${cached ? "cached" : "cloned"} \u2192 ${relPath}`
+        `  Hub nested "${project.name}": ${cached ? "cached" : "cloned"} to ${relPath}`
       );
     } catch (err) {
       console.warn(
-        `  \u26A0 Hub inject "${project.name}" failed:`,
+        `  Hub nested "${project.name}" failed:`,
         err instanceof Error ? err.message : err
       );
     }