@agentmarkup/astro 0.2.0 → 0.3.0

package/README.md

@@ -1,6 +1,6 @@
 # @agentmarkup/astro
 
-Build-time `llms.txt`, JSON-LD, AI crawler `robots.txt`, and validation for Astro websites.
+Build-time `llms.txt`, JSON-LD, markdown mirrors, AI crawler `robots.txt`, headers, and validation for Astro websites.
 
 ## Install
 
@@ -34,6 +34,12 @@ export default defineConfig({
           },
         ],
       },
+      markdownPages: {
+        enabled: true,
+      },
+      contentSignalHeaders: {
+        enabled: true,
+      },
       globalSchemas: [
         {
           preset: 'webSite',
@@ -82,12 +88,18 @@ export default defineConfig({
 
 - Injects JSON-LD into built HTML pages during the Astro build
 - Generates `/llms.txt` from config
+- Validates JSON-LD already present in page HTML
+- Generates `.md` mirrors from the final HTML output
 - Patches or creates `robots.txt` with AI crawler directives
+- Patches or creates `_headers` with `Content-Signal`
 - Validates common schema and crawler mistakes at build time
+- Warns when a page looks like a thin client-rendered HTML shell
 - Re-exports `@agentmarkup/core` helpers for custom pipelines
 
 By default, the Astro adapter coexists with existing machine-readable assets. If a page already contains JSON-LD for a schema type, or the site already ships a curated `llms.txt` or matching crawler rules, those are preserved unless you opt into replacement.
 
+Markdown mirrors and `_headers` follow the same rule: existing files are preserved unless you opt into replacement with `markdownPages.replaceExisting` or `contentSignalHeaders.replaceExisting`.
+
 ## Maintainer
 
 Copyright (c) 2026 [Sebastian Cochinescu](https://www.cochinescu.com). MIT License.

package/dist/index.js

@@ -6,14 +6,21 @@ import { fileURLToPath } from "url";
 import {
   collectSchemasForPage,
   filterJsonLdByExistingTypes,
+  generateMarkdownAlternateLink,
+  generatePageMarkdown,
   generateJsonLdTags,
   generateLlmsTxt,
   hasExistingJsonLdScripts,
+  injectHeadContent,
   injectJsonLdTags,
+  markdownFileNameFromHtmlFile,
+  patchHeadersFile,
   normalizePagePath,
   patchRobotsTxt,
   presetToJsonLd,
   printReport,
+  validateExistingJsonLd,
+  validateHtmlContent,
   validateLlmsTxt,
   validateRobotsTxt,
   validateSchema
@@ -25,8 +32,11 @@ function agentmarkup(config) {
   let llmsTxtSections = 0;
   let llmsTxtStatus = "none";
   let jsonLdPages = 0;
+  let markdownPages = 0;
+  let markdownPagesStatus = "none";
   let crawlersConfigured = 0;
   let robotsTxtStatus = "none";
+  let contentSignalHeadersStatus = "none";
   let publicDir;
   return {
     name: "agentmarkup",
@@ -40,8 +50,19 @@ function agentmarkup(config) {
         for (const htmlFile of htmlFiles) {
           const pagePath = pagePathFromOutputFile(outDir, htmlFile);
           const html = await readFile(htmlFile, "utf8");
+          let nextHtml = html;
           const schemas = collectSchemasForPage(config, pagePath);
-          const hasExistingJsonLd = hasExistingJsonLdScripts(html);
+          const hasExistingJsonLd = hasExistingJsonLdScripts(nextHtml);
+          if (!config.validation?.disabled) {
+            validationResults.push(...validateHtmlContent(nextHtml, pagePath));
+            validationResults.push(...validateExistingJsonLd(nextHtml, pagePath));
+          }
+          if (isFeatureEnabled(config.markdownPages) && pagePath && !hasMarkdownAlternateLink(nextHtml)) {
+            nextHtml = injectHeadContent(
+              nextHtml,
+              generateMarkdownAlternateLink(pagePath)
+            );
+          }
           if (schemas.length === 0) {
             if (pagePath && config.validation?.warnOnMissingSchema && !config.validation.disabled && !hasExistingJsonLd) {
               validationResults.push({
@@ -50,22 +71,59 @@ function agentmarkup(config) {
                 path: pagePath
               });
             }
+            if (nextHtml !== html) {
+              await writeFile(htmlFile, nextHtml, "utf8");
+            }
             continue;
           }
           const jsonLdObjects = schemas.map(presetToJsonLd);
-          const injectables = config.jsonLd?.replaceExistingTypes ? jsonLdObjects : filterJsonLdByExistingTypes(jsonLdObjects, html);
+          const injectables = config.jsonLd?.replaceExistingTypes ? jsonLdObjects : filterJsonLdByExistingTypes(jsonLdObjects, nextHtml);
           if (!config.validation?.disabled) {
             for (const schema of schemas) {
               validationResults.push(...validateSchema(schema, pagePath));
             }
           }
           if (injectables.length === 0) {
+            if (nextHtml !== html) {
+              await writeFile(htmlFile, nextHtml, "utf8");
+            }
             continue;
           }
           const tags = generateJsonLdTags(injectables);
-          await writeFile(htmlFile, injectJsonLdTags(html, tags), "utf8");
+          nextHtml = injectJsonLdTags(nextHtml, tags);
+          await writeFile(htmlFile, nextHtml, "utf8");
           jsonLdPages += 1;
         }
+        if (isFeatureEnabled(config.markdownPages)) {
+          let preservedMarkdownPages = 0;
+          for (const htmlFile of htmlFiles) {
+            const relativeHtmlPath = relative(outDir, htmlFile).replace(/\\/g, "/");
+            const markdownFileName = markdownFileNameFromHtmlFile(relativeHtmlPath);
+            const outputMarkdownPath = join(outDir, markdownFileName);
+            const html = await readFile(htmlFile, "utf8");
+            const pagePath = pagePathFromOutputFile(outDir, htmlFile);
+            const markdown = generatePageMarkdown({
+              html,
+              pagePath,
+              siteUrl: config.site
+            });
+            if (!markdown) {
+              continue;
+            }
+            const existingOutputMarkdown = await readTextFileIfExists(outputMarkdownPath);
+            const existingMarkdown = existingOutputMarkdown ?? (publicDir ? await readTextFileIfExists(join(publicDir, markdownFileName)) : null);
+            if (existingMarkdown && !config.markdownPages?.replaceExisting) {
+              preservedMarkdownPages += 1;
+              if (!existingOutputMarkdown) {
+                await writeFile(outputMarkdownPath, existingMarkdown, "utf8");
+              }
+              continue;
+            }
+            await writeFile(outputMarkdownPath, markdown, "utf8");
+            markdownPages += 1;
+          }
+          markdownPagesStatus = markdownPages > 0 ? "generated" : preservedMarkdownPages > 0 ? "preserved" : "none";
+        }
         const llmsTxtContent = generateLlmsTxt(config);
         if (llmsTxtContent) {
           const outputLlmsPath = join(outDir, "llms.txt");
@@ -114,14 +172,31 @@ function agentmarkup(config) {
             await writeFile(outputRobotsPath, patched, "utf8");
           }
         }
+        if (isFeatureEnabled(config.contentSignalHeaders)) {
+          const outputHeadersPath = join(outDir, "_headers");
+          const existingOutputHeaders = await readTextFileIfExists(outputHeadersPath);
+          const existingHeaders = existingOutputHeaders ?? (publicDir ? await readTextFileIfExists(join(publicDir, "_headers")) : null);
+          const patchedHeaders = patchHeadersFile(
+            existingHeaders,
+            config.contentSignalHeaders
+          );
+          const preserved = existingHeaders !== null && patchedHeaders === existingHeaders;
+          contentSignalHeadersStatus = preserved ? "preserved" : "generated";
+          if (!preserved || !existingOutputFileExists(outputHeadersPath)) {
+            await writeFile(outputHeadersPath, patchedHeaders, "utf8");
+          }
+        }
         printReport({
           label: "@agentmarkup/astro",
           llmsTxtEntries,
           llmsTxtSections,
           llmsTxtStatus,
           jsonLdPages,
+          markdownPages,
+          markdownPagesStatus,
           crawlersConfigured,
           robotsTxtStatus,
+          contentSignalHeadersStatus,
           validationResults
         });
       }
@@ -157,6 +232,14 @@ function pagePathFromOutputFile(outDir, filePath) {
 function existingOutputFileExists(filePath) {
   return existsSync(filePath);
 }
+function isFeatureEnabled(config) {
+  return Boolean(config && config.enabled !== false);
+}
+function hasMarkdownAlternateLink(html) {
+  return /<link\b[^>]*rel=(['"])alternate\1[^>]*type=(['"])text\/markdown\2/i.test(
+    html
+  );
+}
 export {
   agentmarkup
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentmarkup/astro",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Build-time llms.txt, JSON-LD, and AI crawler controls for Astro",
   "type": "module",
   "license": "MIT",
@@ -36,7 +36,7 @@
     "dist"
   ],
   "dependencies": {
-    "@agentmarkup/core": "0.2.0"
+    "@agentmarkup/core": "0.3.0"
   },
   "peerDependencies": {
     "astro": ">=4.0.0"