@agentmarkup/astro 0.3.3 → 0.4.0

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2025 Sebastian Cochinescu and Anima Felix
+Copyright (c) 2026 Sebastian Cochinescu and Anima Felix
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/README.md

@@ -1,6 +1,8 @@
 # @agentmarkup/astro
 
-Build-time `llms.txt`, JSON-LD, markdown mirrors, AI crawler `robots.txt`, headers, and validation for Astro websites.
+Build-time `llms.txt`, optional `llms-full.txt`, JSON-LD, markdown mirrors, AI crawler `robots.txt`, headers, and validation for Astro websites.
+
+`@agentmarkup/astro` is the Astro adapter in the `agentmarkup` package family. Framework-agnostic helpers live in `@agentmarkup/core`, Vite sites use `@agentmarkup/vite`, and Next.js sites use `@agentmarkup/next`.
 
 ## Install
 
@@ -34,6 +36,9 @@ export default defineConfig({
           },
         ],
       },
+      llmsFullTxt: {
+        enabled: true,
+      },
       markdownPages: {
         enabled: true,
       },
@@ -88,22 +93,27 @@ export default defineConfig({
 
 - Injects JSON-LD into built HTML pages during the Astro build
 - Generates `/llms.txt` from config
+- Generates optional `/llms-full.txt` with inlined same-site markdown content
+- Injects the homepage `llms.txt` discovery link automatically
 - Validates JSON-LD already present in page HTML
-- Generates `.md` mirrors from the final HTML output
+- Generates `.md` mirrors from the final HTML output when a cleaner agent-facing fetch path is useful
 - Patches or creates `robots.txt` with AI crawler directives
 - Patches or creates `_headers` with `Content-Signal` and canonical `Link` headers for markdown mirrors
 - Validates common schema and crawler mistakes at build time
 - Warns when a page looks like a thin client-rendered HTML shell
+- Warns when markdown alternate links or `llms.txt` mirror coverage drift out of sync
 - Re-exports `@agentmarkup/core` helpers for custom pipelines
 
 By default, the Astro adapter coexists with existing machine-readable assets. If a page already contains JSON-LD for a schema type, or the site already ships a curated `llms.txt` or matching crawler rules, those are preserved unless you opt into replacement.
 
-Markdown mirrors stay directly fetchable for agents, while their `_headers` entries point search engines back at the HTML page as canonical. Existing files are still preserved unless you opt into replacement with `markdownPages.replaceExisting` or `contentSignalHeaders.replaceExisting`.
+Markdown mirrors are optional. They are usually most useful for thin, noisy, or client-rendered HTML where the raw page is a weak fetch target for agents. The generated `.md` files stay directly fetchable for agents, while their `_headers` entries point search engines back at the HTML page as canonical. Existing files are still preserved unless you opt into replacement with `markdownPages.replaceExisting` or `contentSignalHeaders.replaceExisting`.
 
 When markdown mirrors are enabled, the adapter also writes canonical `Link` headers for those `.md` files so search engines can keep the HTML route as the preferred indexed URL without making the markdown mirror unavailable to direct fetchers.
 
 When markdown mirrors are enabled, same-site page entries in `llms.txt` automatically point at the generated `.md` mirrors by default. Set `llmsTxt.preferMarkdownMirrors: false` if you want `llms.txt` to keep linking to HTML routes instead.
 
+Enable `llmsFullTxt` when you want a richer companion file for agents that can consume more than the compact `llms.txt` manifest. The generated `llms-full.txt` keeps the same section structure but inlines same-site markdown mirror content when those mirrors exist.
+
 ## Maintainer
 
 Copyright (c) 2026 [Sebastian Cochinescu](https://www.cochinescu.com). MIT License.

package/dist/index.js

@@ -6,10 +6,13 @@ import { fileURLToPath } from "url";
 import {
   collectSchemasForPage,
   filterJsonLdByExistingTypes,
+  generateLlmsFullTxt,
+  generateLlmsTxtDiscoveryLink,
   generateMarkdownAlternateLink,
   generatePageMarkdown,
   generateJsonLdTags,
   generateLlmsTxt,
+  hasLlmsTxtDiscoveryLink,
   hasExistingJsonLdScripts,
   injectHeadContent,
   injectJsonLdTags,
@@ -20,9 +23,13 @@ import {
   patchRobotsTxt,
   presetToJsonLd,
   printReport,
+  resolveLlmsTxtSections,
   validateExistingJsonLd,
   validateHtmlContent,
   validateLlmsTxt,
+  validateLlmsTxtMarkdownCoverage,
+  validateMarkdownAlternateLink,
+  validateMarkdownContent,
   validateRobotsTxt,
   validateSchema
 } from "@agentmarkup/core";
@@ -32,6 +39,8 @@ function agentmarkup(config) {
   let llmsTxtEntries = 0;
   let llmsTxtSections = 0;
   let llmsTxtStatus = "none";
+  let llmsFullTxtEntries = 0;
+  let llmsFullTxtStatus = "none";
   let jsonLdPages = 0;
   let markdownPages = 0;
   let markdownPagesStatus = "none";
@@ -50,6 +59,11 @@ function agentmarkup(config) {
       "astro:build:done": async ({ dir }) => {
         const outDir = fileURLToPath(dir);
         const htmlFiles = await findHtmlFiles(outDir);
+        const resolvedLlmsSections = resolveLlmsTxtSections(config);
+        const markdownByUrl = {};
+        const availableMarkdownUrls = /* @__PURE__ */ new Set();
+        const finalHtmlByFile = /* @__PURE__ */ new Map();
+        const advertiseLlmsTxt = Boolean(config.llmsTxt) || Boolean(publicDir && existsSync(join(publicDir, "llms.txt")));
         for (const htmlFile of htmlFiles) {
           const pagePath = pagePathFromOutputFile(outDir, htmlFile);
           const html = await readFile(htmlFile, "utf8");
@@ -60,12 +74,18 @@ function agentmarkup(config) {
             validationResults.push(...validateHtmlContent(nextHtml, pagePath));
             validationResults.push(...validateExistingJsonLd(nextHtml, pagePath));
           }
+          if (advertiseLlmsTxt && !hasLlmsTxtDiscoveryLink(nextHtml)) {
+            nextHtml = injectHeadContent(nextHtml, generateLlmsTxtDiscoveryLink());
+          }
           if (isFeatureEnabled(config.markdownPages) && pagePath && !hasMarkdownAlternateLink(nextHtml)) {
             nextHtml = injectHeadContent(
               nextHtml,
               generateMarkdownAlternateLink(pagePath)
             );
           }
+          if (isFeatureEnabled(config.markdownPages) && !config.validation?.disabled) {
+            validationResults.push(...validateMarkdownAlternateLink(nextHtml, pagePath));
+          }
           if (schemas.length === 0) {
             if (pagePath && config.validation?.warnOnMissingSchema && !config.validation.disabled && !hasExistingJsonLd) {
               validationResults.push({
@@ -77,6 +97,7 @@ function agentmarkup(config) {
             if (nextHtml !== html) {
               await writeFile(htmlFile, nextHtml, "utf8");
             }
+            finalHtmlByFile.set(htmlFile, nextHtml);
             continue;
           }
           const jsonLdObjects = schemas.map(presetToJsonLd);
@@ -90,11 +111,13 @@ function agentmarkup(config) {
             if (nextHtml !== html) {
               await writeFile(htmlFile, nextHtml, "utf8");
             }
+            finalHtmlByFile.set(htmlFile, nextHtml);
             continue;
           }
           const tags = generateJsonLdTags(injectables);
           nextHtml = injectJsonLdTags(nextHtml, tags);
           await writeFile(htmlFile, nextHtml, "utf8");
+          finalHtmlByFile.set(htmlFile, nextHtml);
           jsonLdPages += 1;
         }
         if (isFeatureEnabled(config.markdownPages)) {
@@ -104,8 +127,9 @@ function agentmarkup(config) {
             const relativeHtmlPath = relative(outDir, htmlFile).replace(/\\/g, "/");
             const markdownFileName = markdownFileNameFromHtmlFile(relativeHtmlPath);
             const outputMarkdownPath = join(outDir, markdownFileName);
-            const html = await readFile(htmlFile, "utf8");
+            const html = finalHtmlByFile.get(htmlFile) ?? await readFile(htmlFile, "utf8");
             const pagePath = pagePathFromOutputFile(outDir, htmlFile);
+            const markdownAbsoluteUrl = buildAbsoluteMarkdownUrl(config.site, pagePath);
             const markdown = generatePageMarkdown({
               html,
               pagePath,
@@ -118,20 +142,32 @@ function agentmarkup(config) {
             const existingMarkdown = existingOutputMarkdown ?? (publicDir ? await readTextFileIfExists(join(publicDir, markdownFileName)) : null);
             if (existingMarkdown && !config.markdownPages?.replaceExisting) {
               preservedMarkdownPages += 1;
+              markdownByUrl[markdownAbsoluteUrl] = existingMarkdown;
+              availableMarkdownUrls.add(markdownAbsoluteUrl);
               markdownCanonicalEntries.push({
                 markdownPath: `/${markdownFileName}`,
                 canonicalUrl: buildCanonicalUrl(config.site, pagePath)
               });
+              if (!config.validation?.disabled) {
+                validationResults.push(
+                  ...validateMarkdownContent(existingMarkdown, pagePath)
+                );
+              }
               if (!existingOutputMarkdown) {
                 await writeFile(outputMarkdownPath, existingMarkdown, "utf8");
               }
               continue;
             }
             await writeFile(outputMarkdownPath, markdown, "utf8");
+            markdownByUrl[markdownAbsoluteUrl] = markdown;
+            availableMarkdownUrls.add(markdownAbsoluteUrl);
             markdownCanonicalEntries.push({
               markdownPath: `/${markdownFileName}`,
               canonicalUrl: buildCanonicalUrl(config.site, pagePath)
             });
+            if (!config.validation?.disabled) {
+              validationResults.push(...validateMarkdownContent(markdown, pagePath));
+            }
             markdownPages += 1;
           }
           markdownPagesStatus = markdownPages > 0 ? "generated" : preservedMarkdownPages > 0 ? "preserved" : "none";
@@ -150,6 +186,14 @@ function agentmarkup(config) {
               await writeFile(outputHeadersPath, patchedHeaders, "utf8");
             }
           }
+          if (!config.validation?.disabled && resolvedLlmsSections.length > 0) {
+            validationResults.push(
+              ...validateLlmsTxtMarkdownCoverage(
+                resolvedLlmsSections,
+                availableMarkdownUrls
+              )
+            );
+          }
         }
         const llmsTxtContent = generateLlmsTxt(config);
         if (llmsTxtContent) {
@@ -177,6 +221,36 @@ function agentmarkup(config) {
             }
           }
         }
+        if (config.llmsFullTxt?.enabled) {
+          const llmsFullTxtContent = generateLlmsFullTxt(config, {
+            contentByUrl: markdownByUrl
+          });
+          if (llmsFullTxtContent) {
+            const outputLlmsFullPath = join(outDir, "llms-full.txt");
+            const inlineEntries = countInlinedLlmsFullEntries(
+              resolvedLlmsSections,
+              markdownByUrl
+            );
+            const existingOutputLlmsFull = await readTextFileIfExists(outputLlmsFullPath);
+            const existingLlmsFull = existingOutputLlmsFull ?? (publicDir ? await readTextFileIfExists(join(publicDir, "llms-full.txt")) : null);
+            if (existingLlmsFull && !config.llmsFullTxt.replaceExisting) {
+              llmsFullTxtStatus = "preserved";
+              if (!existingOutputLlmsFull) {
+                await writeFile(outputLlmsFullPath, existingLlmsFull, "utf8");
+              }
+              if (!config.validation?.disabled) {
+                validationResults.push(...validateLlmsTxt(existingLlmsFull));
+              }
+            } else {
+              llmsFullTxtStatus = "generated";
+              llmsFullTxtEntries = inlineEntries;
+              await writeFile(outputLlmsFullPath, llmsFullTxtContent, "utf8");
+              if (!config.validation?.disabled) {
+                validationResults.push(...validateLlmsTxt(llmsFullTxtContent));
+              }
+            }
+          }
+        }
         if (config.aiCrawlers) {
           const crawlerEntries = Object.entries(config.aiCrawlers).filter(
             ([, value]) => value !== void 0
@@ -218,6 +292,8 @@ function agentmarkup(config) {
           llmsTxtEntries,
           llmsTxtSections,
           llmsTxtStatus,
+          llmsFullTxtEntries,
+          llmsFullTxtStatus,
           jsonLdPages,
           markdownPages,
           markdownPagesStatus,
@@ -233,10 +309,14 @@ function agentmarkup(config) {
   };
 }
 async function readTextFileIfExists(filePath) {
-  if (!existsSync(filePath)) {
-    return null;
+  try {
+    return await readFile(filePath, "utf8");
+  } catch (error) {
+    if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
+      return null;
+    }
+    throw error;
   }
-  return readFile(filePath, "utf8");
 }
 async function findHtmlFiles(rootDir) {
   const entries = await readdir(rootDir, { withFileTypes: true });
@@ -262,6 +342,18 @@ function buildCanonicalUrl(siteUrl, pagePath) {
   const base = siteUrl.replace(/\/$/, "");
   return pagePath === "/" ? `${base}/` : `${base}${pagePath}`;
 }
+function buildAbsoluteMarkdownUrl(siteUrl, pagePath) {
+  const base = siteUrl.replace(/\/$/, "");
+  return pagePath === "/" ? `${base}/index.md` : `${base}${pagePath}.md`;
+}
+function countInlinedLlmsFullEntries(sections, markdownByUrl) {
+  return sections.reduce(
+    (sum, section) => sum + section.entries.filter(
+      (entry) => Boolean(entry.markdownUrl && markdownByUrl[entry.markdownUrl])
+    ).length,
+    0
+  );
+}
 function existingOutputFileExists(filePath) {
   return existsSync(filePath);
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@agentmarkup/astro",
-  "version": "0.3.3",
-  "description": "Build-time llms.txt, JSON-LD, markdown mirrors, headers, AI crawler controls, and validation for Astro",
+  "version": "0.4.0",
+  "description": "Build-time llms.txt, llms-full.txt, JSON-LD, markdown mirrors, headers, AI crawler controls, and validation for Astro",
   "type": "module",
   "license": "MIT",
   "author": "Sebastian Cochinescu <hello@animafelix.com> (https://animafelix.com)",
@@ -17,6 +17,7 @@
   "keywords": [
     "astro",
     "llms-txt",
+    "llms-full",
     "json-ld",
     "markdown",
     "schema-org",
@@ -41,7 +42,7 @@
     "dist"
   ],
   "dependencies": {
-    "@agentmarkup/core": "0.3.2"
+    "@agentmarkup/core": "0.4.0"
   },
   "peerDependencies": {
     "astro": ">=4.0.0"