@agentmarkup/astro 0.3.4 → 0.4.0

package/README.md

@@ -2,6 +2,8 @@
 
 Build-time `llms.txt`, optional `llms-full.txt`, JSON-LD, markdown mirrors, AI crawler `robots.txt`, headers, and validation for Astro websites.
 
+`@agentmarkup/astro` is the Astro adapter in the `agentmarkup` package family. Framework-agnostic helpers live in `@agentmarkup/core`, Vite sites use `@agentmarkup/vite`, and Next.js sites use `@agentmarkup/next`.
+
 ## Install
 
 ```bash

package/dist/index.js

@@ -6,10 +6,13 @@ import { fileURLToPath } from "url";
 import {
   collectSchemasForPage,
   filterJsonLdByExistingTypes,
+  generateLlmsFullTxt,
+  generateLlmsTxtDiscoveryLink,
   generateMarkdownAlternateLink,
   generatePageMarkdown,
   generateJsonLdTags,
   generateLlmsTxt,
+  hasLlmsTxtDiscoveryLink,
   hasExistingJsonLdScripts,
   injectHeadContent,
   injectJsonLdTags,
@@ -20,9 +23,13 @@ import {
   patchRobotsTxt,
   presetToJsonLd,
   printReport,
+  resolveLlmsTxtSections,
   validateExistingJsonLd,
   validateHtmlContent,
   validateLlmsTxt,
+  validateLlmsTxtMarkdownCoverage,
+  validateMarkdownAlternateLink,
+  validateMarkdownContent,
   validateRobotsTxt,
   validateSchema
 } from "@agentmarkup/core";
@@ -32,6 +39,8 @@ function agentmarkup(config) {
   let llmsTxtEntries = 0;
   let llmsTxtSections = 0;
   let llmsTxtStatus = "none";
+  let llmsFullTxtEntries = 0;
+  let llmsFullTxtStatus = "none";
   let jsonLdPages = 0;
   let markdownPages = 0;
   let markdownPagesStatus = "none";
@@ -50,6 +59,11 @@ function agentmarkup(config) {
       "astro:build:done": async ({ dir }) => {
         const outDir = fileURLToPath(dir);
         const htmlFiles = await findHtmlFiles(outDir);
+        const resolvedLlmsSections = resolveLlmsTxtSections(config);
+        const markdownByUrl = {};
+        const availableMarkdownUrls = /* @__PURE__ */ new Set();
+        const finalHtmlByFile = /* @__PURE__ */ new Map();
+        const advertiseLlmsTxt = Boolean(config.llmsTxt) || Boolean(publicDir && existsSync(join(publicDir, "llms.txt")));
         for (const htmlFile of htmlFiles) {
           const pagePath = pagePathFromOutputFile(outDir, htmlFile);
           const html = await readFile(htmlFile, "utf8");
@@ -60,12 +74,18 @@ function agentmarkup(config) {
             validationResults.push(...validateHtmlContent(nextHtml, pagePath));
             validationResults.push(...validateExistingJsonLd(nextHtml, pagePath));
           }
+          if (advertiseLlmsTxt && !hasLlmsTxtDiscoveryLink(nextHtml)) {
+            nextHtml = injectHeadContent(nextHtml, generateLlmsTxtDiscoveryLink());
+          }
           if (isFeatureEnabled(config.markdownPages) && pagePath && !hasMarkdownAlternateLink(nextHtml)) {
             nextHtml = injectHeadContent(
               nextHtml,
               generateMarkdownAlternateLink(pagePath)
             );
           }
+          if (isFeatureEnabled(config.markdownPages) && !config.validation?.disabled) {
+            validationResults.push(...validateMarkdownAlternateLink(nextHtml, pagePath));
+          }
           if (schemas.length === 0) {
             if (pagePath && config.validation?.warnOnMissingSchema && !config.validation.disabled && !hasExistingJsonLd) {
               validationResults.push({
@@ -77,6 +97,7 @@ function agentmarkup(config) {
             if (nextHtml !== html) {
               await writeFile(htmlFile, nextHtml, "utf8");
             }
+            finalHtmlByFile.set(htmlFile, nextHtml);
             continue;
           }
           const jsonLdObjects = schemas.map(presetToJsonLd);
@@ -90,11 +111,13 @@ function agentmarkup(config) {
             if (nextHtml !== html) {
               await writeFile(htmlFile, nextHtml, "utf8");
             }
+            finalHtmlByFile.set(htmlFile, nextHtml);
             continue;
           }
           const tags = generateJsonLdTags(injectables);
           nextHtml = injectJsonLdTags(nextHtml, tags);
           await writeFile(htmlFile, nextHtml, "utf8");
+          finalHtmlByFile.set(htmlFile, nextHtml);
           jsonLdPages += 1;
         }
         if (isFeatureEnabled(config.markdownPages)) {
@@ -104,8 +127,9 @@ function agentmarkup(config) {
             const relativeHtmlPath = relative(outDir, htmlFile).replace(/\\/g, "/");
             const markdownFileName = markdownFileNameFromHtmlFile(relativeHtmlPath);
             const outputMarkdownPath = join(outDir, markdownFileName);
-            const html = await readFile(htmlFile, "utf8");
+            const html = finalHtmlByFile.get(htmlFile) ?? await readFile(htmlFile, "utf8");
             const pagePath = pagePathFromOutputFile(outDir, htmlFile);
+            const markdownAbsoluteUrl = buildAbsoluteMarkdownUrl(config.site, pagePath);
             const markdown = generatePageMarkdown({
               html,
               pagePath,
@@ -118,20 +142,32 @@ function agentmarkup(config) {
             const existingMarkdown = existingOutputMarkdown ?? (publicDir ? await readTextFileIfExists(join(publicDir, markdownFileName)) : null);
             if (existingMarkdown && !config.markdownPages?.replaceExisting) {
               preservedMarkdownPages += 1;
+              markdownByUrl[markdownAbsoluteUrl] = existingMarkdown;
+              availableMarkdownUrls.add(markdownAbsoluteUrl);
               markdownCanonicalEntries.push({
                 markdownPath: `/${markdownFileName}`,
                 canonicalUrl: buildCanonicalUrl(config.site, pagePath)
               });
+              if (!config.validation?.disabled) {
+                validationResults.push(
+                  ...validateMarkdownContent(existingMarkdown, pagePath)
+                );
+              }
               if (!existingOutputMarkdown) {
                 await writeFile(outputMarkdownPath, existingMarkdown, "utf8");
               }
               continue;
             }
             await writeFile(outputMarkdownPath, markdown, "utf8");
+            markdownByUrl[markdownAbsoluteUrl] = markdown;
+            availableMarkdownUrls.add(markdownAbsoluteUrl);
             markdownCanonicalEntries.push({
               markdownPath: `/${markdownFileName}`,
               canonicalUrl: buildCanonicalUrl(config.site, pagePath)
             });
+            if (!config.validation?.disabled) {
+              validationResults.push(...validateMarkdownContent(markdown, pagePath));
+            }
             markdownPages += 1;
           }
           markdownPagesStatus = markdownPages > 0 ? "generated" : preservedMarkdownPages > 0 ? "preserved" : "none";
@@ -150,6 +186,14 @@ function agentmarkup(config) {
               await writeFile(outputHeadersPath, patchedHeaders, "utf8");
             }
           }
+          if (!config.validation?.disabled && resolvedLlmsSections.length > 0) {
+            validationResults.push(
+              ...validateLlmsTxtMarkdownCoverage(
+                resolvedLlmsSections,
+                availableMarkdownUrls
+              )
+            );
+          }
         }
         const llmsTxtContent = generateLlmsTxt(config);
         if (llmsTxtContent) {
@@ -177,6 +221,36 @@ function agentmarkup(config) {
             }
           }
         }
+        if (config.llmsFullTxt?.enabled) {
+          const llmsFullTxtContent = generateLlmsFullTxt(config, {
+            contentByUrl: markdownByUrl
+          });
+          if (llmsFullTxtContent) {
+            const outputLlmsFullPath = join(outDir, "llms-full.txt");
+            const inlineEntries = countInlinedLlmsFullEntries(
+              resolvedLlmsSections,
+              markdownByUrl
+            );
+            const existingOutputLlmsFull = await readTextFileIfExists(outputLlmsFullPath);
+            const existingLlmsFull = existingOutputLlmsFull ?? (publicDir ? await readTextFileIfExists(join(publicDir, "llms-full.txt")) : null);
+            if (existingLlmsFull && !config.llmsFullTxt.replaceExisting) {
+              llmsFullTxtStatus = "preserved";
+              if (!existingOutputLlmsFull) {
+                await writeFile(outputLlmsFullPath, existingLlmsFull, "utf8");
+              }
+              if (!config.validation?.disabled) {
+                validationResults.push(...validateLlmsTxt(existingLlmsFull));
+              }
+            } else {
+              llmsFullTxtStatus = "generated";
+              llmsFullTxtEntries = inlineEntries;
+              await writeFile(outputLlmsFullPath, llmsFullTxtContent, "utf8");
+              if (!config.validation?.disabled) {
+                validationResults.push(...validateLlmsTxt(llmsFullTxtContent));
+              }
+            }
+          }
+        }
         if (config.aiCrawlers) {
           const crawlerEntries = Object.entries(config.aiCrawlers).filter(
             ([, value]) => value !== void 0
@@ -218,6 +292,8 @@ function agentmarkup(config) {
           llmsTxtEntries,
           llmsTxtSections,
           llmsTxtStatus,
+          llmsFullTxtEntries,
+          llmsFullTxtStatus,
           jsonLdPages,
           markdownPages,
           markdownPagesStatus,
@@ -233,10 +309,14 @@ function agentmarkup(config) {
   };
 }
 async function readTextFileIfExists(filePath) {
-  if (!existsSync(filePath)) {
-    return null;
+  try {
+    return await readFile(filePath, "utf8");
+  } catch (error) {
+    if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
+      return null;
+    }
+    throw error;
   }
-  return readFile(filePath, "utf8");
 }
 async function findHtmlFiles(rootDir) {
   const entries = await readdir(rootDir, { withFileTypes: true });
@@ -262,6 +342,18 @@ function buildCanonicalUrl(siteUrl, pagePath) {
   const base = siteUrl.replace(/\/$/, "");
   return pagePath === "/" ? `${base}/` : `${base}${pagePath}`;
 }
+function buildAbsoluteMarkdownUrl(siteUrl, pagePath) {
+  const base = siteUrl.replace(/\/$/, "");
+  return pagePath === "/" ? `${base}/index.md` : `${base}${pagePath}.md`;
+}
+function countInlinedLlmsFullEntries(sections, markdownByUrl) {
+  return sections.reduce(
+    (sum, section) => sum + section.entries.filter(
+      (entry) => Boolean(entry.markdownUrl && markdownByUrl[entry.markdownUrl])
+    ).length,
+    0
+  );
+}
 function existingOutputFileExists(filePath) {
   return existsSync(filePath);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentmarkup/astro",
-  "version": "0.3.4",
+  "version": "0.4.0",
   "description": "Build-time llms.txt, llms-full.txt, JSON-LD, markdown mirrors, headers, AI crawler controls, and validation for Astro",
   "type": "module",
   "license": "MIT",
@@ -42,7 +42,7 @@
     "dist"
   ],
   "dependencies": {
-    "@agentmarkup/core": "0.3.4"
+    "@agentmarkup/core": "0.4.0"
   },
   "peerDependencies": {
     "astro": ">=4.0.0"