nuxt-ai-ready 3.3.0 → 3.4.0

package/dist/runtime/server/utils/sitemap-lastmod.d.ts

@@ -0,0 +1,4 @@
+/** Mark dates added by this module without changing the public sitemap date. */
+export declare function markIndexedSitemapLastmods(xml: string, enrichedUrls: ReadonlySet<string>): string;
+/** The source crawl must never treat our indexed date as a CMS change. */
+export declare function stripIndexedSitemapLastmods(body: Uint8Array): Uint8Array;

package/dist/runtime/server/utils/sitemap-lastmod.js

@@ -0,0 +1,19 @@
+const INDEXED_LASTMOD_MARKER = "<!-- nuxt-ai-ready:indexed-lastmod -->";
+export function markIndexedSitemapLastmods(xml, enrichedUrls) {
+  const escapedUrls = new Set([...enrichedUrls].map((url) => url.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&apos;")));
+  return xml.replace(/<url>([\s\S]*?)<\/url>/g, (entry, body) => {
+    const loc = body.match(/<loc>([^<]*)<\/loc>/)?.[1];
+    return loc && escapedUrls.has(loc) ? entry.replace("<lastmod>", `${INDEXED_LASTMOD_MARKER}<lastmod>`) : entry;
+  });
+}
+export function stripIndexedSitemapLastmods(body) {
+  let xml;
+  try {
+    xml = new TextDecoder("utf-8", { fatal: true }).decode(body);
+  } catch {
+    return body;
+  }
+  if (!xml.includes(INDEXED_LASTMOD_MARKER))
+    return body;
+  return new TextEncoder().encode(xml.replace(/<!-- nuxt-ai-ready:indexed-lastmod -->\s*<lastmod>[^<]*<\/lastmod>/g, ""));
+}

package/dist/runtime/server/utils/sitemap-md.js

@@ -1,11 +1,12 @@
 import { toMarkdownPath } from "../../markdown-path.js";
+import { escapeMarkdownHtml } from "../../markdown-text.js";
 import { toDeployedRoute } from "../../route-path.js";
 export const SITEMAP_MD_ROUTE = "/sitemap.md";
 function topSegment(route) {
   return route.split("/").filter(Boolean)[0] || "";
 }
 function escapeTitle(title) {
-  return title.replaceAll("[", "\\[").replaceAll("]", "\\]");
+  return escapeMarkdownHtml(title).replaceAll("[", "\\[").replaceAll("]", "\\]");
 }
 function formatEntry(entry, resolveHref) {
   const title = escapeTitle(entry.title?.trim() || entry.route);
@@ -35,7 +36,7 @@ export function buildSitemapMd(entries, options = {}) {
       return 1;
     return a.localeCompare(b);
   });
-  const parts = [`# ${options.siteName || "Site"} Sitemap`, "", "All pages in Markdown format."];
+  const parts = [`# ${escapeMarkdownHtml(options.siteName || "Site")} Sitemap`, "", "All pages in Markdown format."];
   for (const key of keys) {
     parts.push("", `## ${key || "Root"}`);
     for (const entry of groups.get(key))

package/dist/runtime/server/utils/sitemap.js

@@ -7,6 +7,7 @@ import { logger } from "../logger.js";
 import { getCfEnv, hasAssets } from "./cloudflare.js";
 import { createUniversalContext } from "./context.js";
 import { fetchRawWithEvent } from "./fetch.js";
+import { stripIndexedSitemapLastmods } from "./sitemap-lastmod.js";
 const FETCH_TIMEOUT = 15e3;
 const SITEMAP_FALLBACK_ORIGIN = "http://localhost";
 const SITEMAP_READER_LIMITS = {
@@ -221,11 +222,12 @@ function createLocalSitemapLoader(event, usePublicAsset, loadedBodies) {
         bytesRead: read.result.bytesRead
       };
     }
-    loadedBodies.set(request.url, read.result.body);
+    const sourceBody = stripIndexedSitemapLastmods(read.result.body);
+    loadedBodies.set(request.url, sourceBody);
     return {
       _tag: "body",
       url: request.url,
-      body: read.result.body
+      body: sourceBody
     };
   };
 }

package/dist/runtime/server/utils.js

@@ -3,11 +3,57 @@ import { useNitroApp } from "#nuxtseo/nitro";
 import { stripLlmsTxtDirectiveHtml } from "../llms-txt-directive.js";
 import { hoistMetaDescription } from "./utils/frontmatter.js";
 const RE_NBSP = /\u00A0/g;
+const JSON_LD_MAX_BYTES = 65536;
+const JSON_LD_MAX_BLOCKS = 16;
+const IMAGE_META_FIELDS = [
+  "og:image",
+  "og:image:url",
+  "og:image:secure_url",
+  "og:image:type",
+  "og:image:width",
+  "og:image:height",
+  "og:image:alt",
+  "twitter:image",
+  "twitter:image:src",
+  "twitter:image:alt"
+];
+function parseJsonLdBlock(text, remainingBytes) {
+  if (text.length > remainingBytes)
+    return { _tag: "Skipped" };
+  const content = text.trim();
+  if (!content)
+    return { _tag: "Skipped" };
+  let value;
+  try {
+    value = JSON.parse(text);
+  } catch {
+    return { _tag: "Skipped" };
+  }
+  const isObject = (item) => typeof item === "object" && item !== null && !Array.isArray(item);
+  if (!isObject(value) && !(Array.isArray(value) && value.every(isObject)))
+    return { _tag: "Skipped" };
+  const markdown = `\`\`\`json
+${content.replace(RE_NBSP, "\\u00a0")}
+\`\`\``;
+  const bytes = new TextEncoder().encode(markdown).byteLength + 2;
+  if (bytes > remainingBytes)
+    return { _tag: "Skipped" };
+  return { _tag: "Preserved", markdown, bytes };
+}
 function normalizeWhitespace(text) {
   return text.replace(RE_NBSP, " ");
 }
 function buildMdreamOptions(url, mdreamOptions, meta, extractUpdatedAt, additionalFields) {
   const extraction = {
+    "script[type]": (el) => {
+      if (el.attributes.type?.trim().toLowerCase() !== "application/ld+json" || meta.jsonLd.blocks.length >= JSON_LD_MAX_BLOCKS)
+        return;
+      const block = parseJsonLdBlock(el.textContent, JSON_LD_MAX_BYTES - meta.jsonLd.bytes);
+      if (block._tag === "Preserved") {
+        meta.jsonLd.blocks.push(block.markdown);
+        meta.jsonLd.bytes += block.bytes;
+      }
+    },
     'meta[name="keywords"]': (el) => {
       meta.metaKeywords = el.attributes.content || "";
     },
@@ -30,6 +76,7 @@ function buildMdreamOptions(url, mdreamOptions, meta, extractUpdatedAt, addition
   };
   const frontmatter = {
     additionalFields,
+    metaFields: [...IMAGE_META_FIELDS],
     onExtract: (fm) => {
       if (fm.title)
         meta.title = fm.title;
@@ -70,13 +117,14 @@ export function extractLastUpdated(html) {
   return void 0;
 }
 export async function convertHtmlToMarkdown(html, url, mdreamOptions, opts) {
-  const meta = { title: "", description: "", metaKeywords: "", headings: [], textContent: [] };
+  const meta = { title: "", description: "", metaKeywords: "", headings: [], textContent: [], jsonLd: { blocks: [], bytes: 0 } };
   const options = buildMdreamOptions(url, mdreamOptions, meta, opts.extractUpdatedAt ?? false, opts.additionalFrontmatter ?? {});
   const nitroApp = useNitroApp();
   await nitroApp.hooks.callHook("ai-ready:mdreamConfig", options);
+  const convertedMarkdown = hoistMetaDescription(htmlToMarkdown(stripLlmsTxtDirectiveHtml(html), options));
   const context = {
     html,
-    markdown: hoistMetaDescription(htmlToMarkdown(stripLlmsTxtDirectiveHtml(html), options)),
+    markdown: [convertedMarkdown, ...meta.jsonLd.blocks].join("\n\n"),
     route: opts.hooks.route,
     title: meta.title,
     description: meta.description,