@marlinjai/clearify 2.0.0 → 2.1.0

package/dist/node/{chunk-CK2SLBAG.js → chunk-GZE6QLJT.js}

@@ -2,7 +2,7 @@ import {
   loadUserConfig,
   resolveConfig,
   resolveSections
-} from "./chunk-PGTNVDNF.js";
+} from "./chunk-ZWF6VCSQ.js";
 import {
   scanDocs
 } from "./chunk-CTL4YL7C.js";

package/dist/node/{chunk-BCEGDIR6.js → chunk-PZA4UAFG.js}

@@ -6,14 +6,18 @@ import {
 import {
   mermaidContentHash
 } from "./chunk-SCZZB7OE.js";
+import {
+  generateLlmsFullTxt,
+  generateLlmsTxt
+} from "./chunk-TVDWHKQF.js";
 import {
   clearifyPlugin
-} from "./chunk-RW4IKHY4.js";
+} from "./chunk-Q5FUAK6J.js";
 import {
   loadUserConfig,
   resolveConfig,
   resolveSections
-} from "./chunk-PGTNVDNF.js";
+} from "./chunk-ZWF6VCSQ.js";
 import {
   buildSectionData
 } from "./chunk-CTL4YL7C.js";
@@ -72,6 +76,29 @@ function generateSitemap(routes, siteUrl) {
 ${urls.join("\n")}
 </urlset>`;
 }
+function formatBytes(bytes) {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
+}
+function collectLlmsDocs(sections) {
+  const out = [];
+  for (const sd of sections) {
+    for (const doc of sd.docs) {
+      const title = doc.frontmatter.title ?? "Untitled";
+      const optional = (doc.frontmatter.order ?? 0) >= 9e3;
+      const markdown = existsSync(doc.filePath) ? readFileSync(doc.filePath, "utf-8") : "";
+      out.push({
+        title,
+        routePath: doc.routePath,
+        sectionLabel: sd.section.label,
+        markdown,
+        optional
+      });
+    }
+  }
+  return out;
+}
 function generateRobotsTxt(siteUrl) {
   const lines = ["User-agent: *", "Allow: /"];
   if (siteUrl) {
@@ -312,6 +339,19 @@ async function buildSite() {
   const sitemap = generateSitemap(sitemapRoutes, config.siteUrl);
   writeFileSync(resolve(outDir, "sitemap.xml"), sitemap);
   console.log("  Generated sitemap.xml");
+  if (config.generateLlmsTxt !== false) {
+    const llmsDocs = collectLlmsDocs(sectionDataList);
+    const llmsTxt = generateLlmsTxt(config, llmsDocs);
+    const llmsFullTxt = generateLlmsFullTxt(config, llmsDocs);
+    const llmsTxtPath = resolve(outDir, "llms.txt");
+    const llmsFullTxtPath = resolve(outDir, "llms-full.txt");
+    writeFileSync(llmsTxtPath, llmsTxt);
+    writeFileSync(llmsFullTxtPath, llmsFullTxt);
+    const llmsTxtSize = Buffer.byteLength(llmsTxt, "utf-8");
+    const llmsFullSize = Buffer.byteLength(llmsFullTxt, "utf-8");
+    console.log(`  Generated ${llmsTxtPath} (${formatBytes(llmsTxtSize)})`);
+    console.log(`  Generated ${llmsFullTxtPath} (${formatBytes(llmsFullSize)})`);
+  }
   rmSync(ssrOutDir, { recursive: true, force: true });
   console.log("  Cleaned up SSR build artifacts.");
   if (mermaidStrategy === "build") {

package/dist/node/{chunk-RW4IKHY4.js → chunk-Q5FUAK6J.js}

@@ -10,7 +10,7 @@ import {
   resolveSections,
   scanHubProjects,
   writeDataConfig
-} from "./chunk-PGTNVDNF.js";
+} from "./chunk-ZWF6VCSQ.js";
 import {
   buildSectionData
 } from "./chunk-CTL4YL7C.js";

package/dist/node/chunk-TVDWHKQF.js

@@ -0,0 +1,134 @@
+// src/core/llms-txt.ts
+import { readFileSync } from "fs";
+import matter from "gray-matter";
+function normalizeBase(siteUrl) {
+  if (!siteUrl) return "";
+  return siteUrl.replace(/\/$/, "");
+}
+function toAbsoluteUrl(base, routePath) {
+  if (!base) return routePath;
+  if (!routePath.startsWith("/")) return `${base}/${routePath}`;
+  return `${base}${routePath}`;
+}
+function stripFrontmatterAndH1(markdown) {
+  const { content } = matter(markdown);
+  return content.replace(/^\s*#\s+[^\n]*\n+/, "").trimStart();
+}
+function firstParagraphSummary(markdown) {
+  const parsed = matter(markdown);
+  const fromFrontmatter = typeof parsed.data.summary === "string" && parsed.data.summary.trim() || typeof parsed.data.description === "string" && parsed.data.description.trim() || "";
+  const body = parsed.content;
+  const lines = body.split(/\r?\n/);
+  let inFence = false;
+  const paragraph = [];
+  for (const rawLine of lines) {
+    const line = rawLine.trimEnd();
+    if (/^```/.test(line)) {
+      inFence = !inFence;
+      if (paragraph.length > 0) break;
+      continue;
+    }
+    if (inFence) continue;
+    if (!line.trim()) {
+      if (paragraph.length > 0) break;
+      continue;
+    }
+    if (/^(#{1,6}\s|>\s|[-*+]\s|\d+\.\s|\|.*\|\s*$|<)/.test(line)) {
+      if (paragraph.length > 0) break;
+      continue;
+    }
+    paragraph.push(line.trim());
+  }
+  const text = paragraph.join(" ").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/`([^`]+)`/g, "$1").replace(/([*_]{1,3})([^*_]+)\1/g, "$2").replace(/\s+/g, " ").trim();
+  return text || fromFrontmatter;
+}
+function uniqueOrdered(items) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const item of items) {
+    if (seen.has(item)) continue;
+    seen.add(item);
+    out.push(item);
+  }
+  return out;
+}
+function buildHeader(config) {
+  const name = config.name.trim();
+  const tagline = config.hubProject?.description && config.hubProject.description.trim() || `Documentation for ${name}.`;
+  return `# ${name}
+
+> ${tagline}
+
+`;
+}
+function generateLlmsTxt(config, docs) {
+  const base = normalizeBase(config.siteUrl);
+  let out = buildHeader(config);
+  const mainDocs = docs.filter((d) => !d.optional);
+  const optionalDocs = docs.filter((d) => d.optional);
+  const sectionLabels = uniqueOrdered(mainDocs.map((d) => d.sectionLabel));
+  for (const label of sectionLabels) {
+    const sectionDocs = mainDocs.filter((d) => d.sectionLabel === label);
+    if (sectionDocs.length === 0) continue;
+    out += `## ${label}
+
+`;
+    for (const doc of sectionDocs) {
+      const url = toAbsoluteUrl(base, doc.routePath);
+      const summary = firstParagraphSummary(doc.markdown);
+      if (summary) {
+        out += `- [${doc.title}](${url}): ${summary}
+`;
+      } else {
+        out += `- [${doc.title}](${url})
+`;
+      }
+    }
+    out += "\n";
+  }
+  if (optionalDocs.length > 0) {
+    out += `## Optional
+
+`;
+    for (const doc of optionalDocs) {
+      const url = toAbsoluteUrl(base, doc.routePath);
+      const summary = firstParagraphSummary(doc.markdown);
+      if (summary) {
+        out += `- [${doc.title}](${url}): ${summary}
+`;
+      } else {
+        out += `- [${doc.title}](${url})
+`;
+      }
+    }
+    out += "\n";
+  }
+  return out.trimEnd() + "\n";
+}
+function generateLlmsFullTxt(config, docs) {
+  const base = normalizeBase(config.siteUrl);
+  let out = buildHeader(config);
+  for (const doc of docs) {
+    const url = toAbsoluteUrl(base, doc.routePath);
+    const body = stripFrontmatterAndH1(doc.markdown);
+    out += `## ${doc.title}
+
+`;
+    out += `Source: ${url}
+
+`;
+    if (body) {
+      out += body.trimEnd() + "\n\n";
+    }
+  }
+  return out.trimEnd() + "\n";
+}
+function readMarkdownFile(filePath) {
+  return readFileSync(filePath, "utf-8");
+}
+
+export {
+  generateLlmsTxt,
+  generateLlmsFullTxt,
+  readMarkdownFile
+};

package/dist/node/{chunk-PGTNVDNF.js → chunk-ZWF6VCSQ.js}

@@ -117,7 +117,8 @@ var ClearifyConfigSchema = z.object({
   hubProject: HubProjectPartialSchema,
   customCss: z.string().optional(),
   headTags: z.array(z.string()).optional(),
-  includeReadme: z.boolean().default(false)
+  includeReadme: z.boolean().default(false),
+  generateLlmsTxt: z.boolean().default(true)
 });
 var ClearifyDataSchema = z.object({
   name: z.string().optional(),
@@ -265,6 +266,11 @@ function resolveConfig(userConfig = {}, root) {
   if (!userConfig.name && root) {
     parsed.name = detectProjectName(root);
   }
+  if (parsed.generateLlmsTxt && !parsed.siteUrl) {
+    throw new Error(
+      "clearify config: `generateLlmsTxt` is enabled (default) but `siteUrl` is not set. Set `siteUrl` to the production URL of your docs site, or set `generateLlmsTxt: false` to disable."
+    );
+  }
   return parsed;
 }
 function slugify(str) {

package/dist/node/core/config.js

@@ -8,7 +8,7 @@ import {
   resolveSections,
   scanHubProjects,
   writeDataConfig
-} from "../chunk-PGTNVDNF.js";
+} from "../chunk-ZWF6VCSQ.js";
 import "../chunk-AJBCHKBP.js";
 import {
   defineConfig

package/dist/node/core/llms-txt.d.ts

@@ -0,0 +1,54 @@
+import { ClearifyConfig } from '../types/index.js';
+
+/**
+ * A doc entry as consumed by the llms.txt generators. Produced from the
+ * scanned section data by callers (e.g. the build pipeline) so these
+ * functions stay pure string builders.
+ */
+interface LlmsDocEntry {
+    /** Page title (from frontmatter or derived). */
+    title: string;
+    /** Route path on the site, e.g. `/getting-started`. */
+    routePath: string;
+    /** Section label the page belongs to, e.g. `Documentation`. */
+    sectionLabel: string;
+    /** Raw markdown source (frontmatter optional, handled by the generator). */
+    markdown: string;
+    /**
+     * Optional flag. When true the page goes under the `## Optional` bucket at
+     * the end of llms.txt rather than the main section list. Use for auxiliary
+     * content like a changelog or roadmap.
+     */
+    optional?: boolean;
+}
+/**
+ * Generate the `llms.txt` index file content.
+ *
+ * Format:
+ *   # {name}
+ *
+ *   > {tagline}
+ *
+ *   ## {Section label}
+ *
+ *   - [Title](https://site/path): first-paragraph summary
+ *   ...
+ *
+ *   ## Optional
+ *   - [Title](https://site/path): summary
+ */
+declare function generateLlmsTxt(config: ClearifyConfig, docs: LlmsDocEntry[]): string;
+/**
+ * Generate the `llms-full.txt` bundle file content. Concatenates the entire
+ * markdown body of every doc under an H2 with a `Source:` line, so an LLM
+ * with no browsing can ingest the whole site from one file.
+ */
+declare function generateLlmsFullTxt(config: ClearifyConfig, docs: LlmsDocEntry[]): string;
+/**
+ * Read a markdown file from disk. Returned as-is, including frontmatter.
+ * Exists here so callers in the build pipeline stay thin and the unit tests
+ * can stub in-memory markdown without touching the filesystem.
+ */
+declare function readMarkdownFile(filePath: string): string;
+
+export { type LlmsDocEntry, generateLlmsFullTxt, generateLlmsTxt, readMarkdownFile };

package/dist/node/core/llms-txt.js

@@ -0,0 +1,10 @@
+import {
+  generateLlmsFullTxt,
+  generateLlmsTxt,
+  readMarkdownFile
+} from "../chunk-TVDWHKQF.js";
+export {
+  generateLlmsFullTxt,
+  generateLlmsTxt,
+  readMarkdownFile
+};

package/dist/node/node/build.js

@@ -1,11 +1,12 @@
 import {
   buildSite
-} from "../chunk-BCEGDIR6.js";
+} from "../chunk-PZA4UAFG.js";
 import "../chunk-PRTER35L.js";
 import "../chunk-SCZZB7OE.js";
-import "../chunk-RW4IKHY4.js";
+import "../chunk-TVDWHKQF.js";
+import "../chunk-Q5FUAK6J.js";
 import "../chunk-B2Q23JW3.js";
-import "../chunk-PGTNVDNF.js";
+import "../chunk-ZWF6VCSQ.js";
 import "../chunk-AJBCHKBP.js";
 import "../chunk-V7LLYIRO.js";
 import "../chunk-CTL4YL7C.js";

package/dist/node/node/check.js

@@ -1,8 +1,8 @@
 import {
   checkFrontmatter,
   checkLinks
-} from "../chunk-CK2SLBAG.js";
-import "../chunk-PGTNVDNF.js";
+} from "../chunk-GZE6QLJT.js";
+import "../chunk-ZWF6VCSQ.js";
 import "../chunk-AJBCHKBP.js";
 import "../chunk-V7LLYIRO.js";
 import "../chunk-CTL4YL7C.js";

package/dist/node/node/index.js

@@ -1,13 +1,14 @@
 import {
   buildSite
-} from "../chunk-BCEGDIR6.js";
+} from "../chunk-PZA4UAFG.js";
 import {
   remarkMermaidToComponent
 } from "../chunk-PRTER35L.js";
 import "../chunk-SCZZB7OE.js";
+import "../chunk-TVDWHKQF.js";
 import {
   clearifyPlugin
-} from "../chunk-RW4IKHY4.js";
+} from "../chunk-Q5FUAK6J.js";
 import "../chunk-B2Q23JW3.js";
 import {
   init
@@ -15,11 +16,11 @@ import {
 import {
   checkFrontmatter,
   checkLinks
-} from "../chunk-CK2SLBAG.js";
+} from "../chunk-GZE6QLJT.js";
 import {
   loadUserConfig,
   resolveConfig
-} from "../chunk-PGTNVDNF.js";
+} from "../chunk-ZWF6VCSQ.js";
 import "../chunk-AJBCHKBP.js";
 import "../chunk-V7LLYIRO.js";
 import "../chunk-CTL4YL7C.js";

package/dist/node/types/index.d.ts

@@ -62,6 +62,12 @@ interface ClearifyConfig {
     customCss?: string;
     headTags?: string[];
     includeReadme?: boolean;
+    /**
+     * When true (default), `clearify build` writes `llms.txt` and
+     * `llms-full.txt` into the output directory for AI agents and crawlers.
+     * Requires `siteUrl` to be set. Set to `false` to skip generation.
+     */
+    generateLlmsTxt?: boolean;
 }
 interface NavigationItem {
     label: string;

package/dist/node/vite-plugin/index.js

@@ -1,8 +1,8 @@
 import {
   clearifyPlugin
-} from "../chunk-RW4IKHY4.js";
+} from "../chunk-Q5FUAK6J.js";
 import "../chunk-B2Q23JW3.js";
-import "../chunk-PGTNVDNF.js";
+import "../chunk-ZWF6VCSQ.js";
 import "../chunk-AJBCHKBP.js";
 import "../chunk-V7LLYIRO.js";
 import "../chunk-CTL4YL7C.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@marlinjai/clearify",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "An open-source documentation site generator. Turn markdown into beautiful docs.",
   "type": "module",
   "bin": {