nuxt-ai-ready 0.3.7 → 0.4.0

package/dist/runtime/{llms-txt.d.ts → llms-txt-utils.d.ts}

@@ -1,5 +1,7 @@
+import type { H3Event } from 'h3';
 import type { LlmsTxtConfig } from './types.js';
 /**
  * Normalize llms.txt structured configuration to markdown string
  */
 export declare function normalizeLlmsTxtConfig(config: LlmsTxtConfig): string;
+export declare function buildLlmsTxt(event: H3Event): Promise<string>;

package/dist/runtime/llms-txt-utils.js

@@ -0,0 +1,114 @@
+import { getSiteConfig } from "#site-config/server/composables/getSiteConfig";
+import { useRuntimeConfig } from "nitropack/runtime";
+import { getPages } from "./server/utils/pageData.js";
+import { fetchSitemapUrls } from "./server/utils/sitemap.js";
+function normalizeLink(link) {
+  const parts = [];
+  parts.push(`- [${link.title}](${link.href})`);
+  if (link.description) {
+    parts.push(`  ${link.description}`);
+  }
+  return parts.join("\n");
+}
+function normalizeSection(section) {
+  const parts = [];
+  parts.push(`## ${section.title}`);
+  parts.push("");
+  if (section.description) {
+    const descriptions = Array.isArray(section.description) ? section.description : [section.description];
+    parts.push(...descriptions);
+    parts.push("");
+  }
+  if (section.links?.length) {
+    parts.push(...section.links.map(normalizeLink));
+  }
+  return parts.join("\n");
+}
+export function normalizeLlmsTxtConfig(config) {
+  const parts = [];
+  if (config.sections?.length) {
+    parts.push(...config.sections.map(normalizeSection));
+  }
+  if (config.notes) {
+    parts.push("## Notes");
+    parts.push("");
+    const notes = Array.isArray(config.notes) ? config.notes : [config.notes];
+    parts.push(...notes);
+  }
+  return parts.join("\n\n");
+}
+export async function buildLlmsTxt(event) {
+  const runtimeConfig = useRuntimeConfig(event);
+  const aiReadyConfig = runtimeConfig["nuxt-ai-ready"];
+  const sitemapConfig = runtimeConfig.sitemap;
+  const siteConfig = getSiteConfig(event);
+  const llmsTxtConfig = aiReadyConfig.llmsTxt;
+  const parts = [];
+  parts.push(`# ${siteConfig.name || siteConfig.url}`);
+  if (siteConfig.description) {
+    parts.push(`
+> ${siteConfig.description}`);
+  }
+  if (siteConfig.url) {
+    parts.push(`
+Canonical Origin: ${siteConfig.url}`);
+  }
+  parts.push("");
+  const sections = llmsTxtConfig.sections ? [...llmsTxtConfig.sections] : [];
+  if (sections[0]?.links) {
+    if (sitemapConfig?.sitemaps) {
+      const sitemapRoutes = Object.values(sitemapConfig.sitemaps).map((s) => s.sitemapName);
+      for (const name of sitemapRoutes) {
+        sections[0].links.push({ title: name, href: `/${name}`, description: "XML sitemap for search engines and crawlers." });
+      }
+    }
+    sections[0].links.push({ title: "robots.txt", href: "/robots.txt", description: "Crawler rules and permissions." });
+  }
+  const normalizedContent = normalizeLlmsTxtConfig({ ...llmsTxtConfig, sections });
+  if (normalizedContent) {
+    parts.push(normalizedContent);
+    parts.push("");
+  }
+  const pages = await getPages();
+  const urls = await fetchSitemapUrls(event);
+  const devModeHint = import.meta.dev && pages.size === 0 ? " (dev mode - run `nuxi generate` for page titles)" : "";
+  const prerendered = [];
+  for (const [pathname, page] of pages) {
+    prerendered.push({ pathname, title: page.title });
+  }
+  const other = [];
+  for (const url of urls) {
+    const pathname = url.loc.startsWith("http") ? new URL(url.loc).pathname : url.loc;
+    if (!pages.has(pathname)) {
+      other.push(pathname);
+    }
+  }
+  if (prerendered.length > 0 && other.length > 0) {
+    parts.push(`## Prerendered Pages${devModeHint}
+`);
+    for (const { pathname, title } of prerendered) {
+      parts.push(title && title !== pathname ? `- [${title}](${pathname})` : `- ${pathname}`);
+    }
+    parts.push("");
+    parts.push("## Other Pages\n");
+    for (const pathname of other) {
+      parts.push(`- ${pathname}`);
+    }
+    parts.push("");
+  } else if (prerendered.length > 0) {
+    parts.push(`## Pages${devModeHint}
+`);
+    for (const { pathname, title } of prerendered) {
+      parts.push(title && title !== pathname ? `- [${title}](${pathname})` : `- ${pathname}`);
+    }
+    parts.push("");
+  } else if (other.length > 0) {
+    parts.push(`## Pages${devModeHint}
+`);
+    for (const pathname of other) {
+      parts.push(`- ${pathname}`);
+    }
+    parts.push("");
+  }
+  return parts.join("\n");
+}

package/dist/runtime/nuxt/plugins/{prerender.js → md-hints.prerender.js}

@@ -7,7 +7,7 @@ export default defineNuxtPlugin({
     }
     nuxtApp.hooks.hook("app:rendered", (ctx) => {
       let url = ctx.ssrContext?.url || "";
-      if (isPathFile(url)) {
+      if (isPathFile(url) || ctx.ssrContext?.error || ctx.ssrContext?.noSSR) {
         return;
       }
       if (url.endsWith("/")) {

package/dist/runtime/server/mcp/resources/pages.d.ts

@@ -0,0 +1,17 @@
+declare const _default: {
+    uri: string;
+    name: string;
+    description: string;
+    metadata: {
+        mimeType: string;
+    };
+    cache: "1h";
+    handler(uri: URL): Promise<{
+        contents: {
+            uri: string;
+            mimeType: string;
+            text: string;
+        }[];
+    }>;
+};
+export default _default;

package/dist/runtime/server/mcp/{dev/resources → resources}/pages.js

@@ -1,18 +1,19 @@
-import { getDevPages } from "../utils.js";
+import { getPagesList } from "../../utils/pageData.js";
 export default {
   uri: "resource://nuxt-ai-ready/pages",
   name: "All Pages",
-  description: "Page routes from sitemap/routes. In dev mode, returns JSON (TOON format unavailable until build).",
+  description: "Page listing as JSON.",
   metadata: {
     mimeType: "application/json"
   },
+  cache: "1h",
   async handler(uri) {
-    const pages = await getDevPages();
+    const pages = await getPagesList();
     return {
       contents: [{
         uri: uri.toString(),
         mimeType: "application/json",
-        text: JSON.stringify(pages, null, 2)
+        text: JSON.stringify(pages)
       }]
     };
   }

package/dist/runtime/server/mcp/tools/list-pages.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Lists all pages with metadata
+ */
+declare const _default: {
+    name: string;
+    description: string;
+    inputSchema: {};
+    cache: "1h";
+    handler(): Promise<{
+        content: {
+            type: "text";
+            text: string;
+        }[];
+    }>;
+};
+export default _default;

package/dist/runtime/server/mcp/tools/list-pages.js

@@ -0,0 +1,11 @@
+import { getPagesList } from "../../utils/pageData.js";
+export default {
+  name: "list_pages",
+  description: "Lists all available pages with titles, descriptions and routes.",
+  inputSchema: {},
+  cache: "1h",
+  async handler() {
+    const pages = await getPagesList();
+    return { content: [{ type: "text", text: JSON.stringify(pages) }] };
+  }
+};

package/dist/runtime/server/mcp/tools/search-pages-fuzzy.d.ts

@@ -0,0 +1,3 @@
+import type { McpToolDefinition } from '@nuxtjs/mcp-toolkit';
+declare const tool: McpToolDefinition;
+export default tool;

package/dist/runtime/server/mcp/tools/search-pages-fuzzy.js

@@ -0,0 +1,25 @@
+import Fuse from "fuse.js";
+import { z } from "zod";
+import { getPagesList } from "../../utils/pageData.js";
+const inputSchema = {
+  query: z.string().describe("Search query"),
+  limit: z.number().optional().default(10).describe("Max results")
+};
+const tool = {
+  name: "search_pages_fuzzy",
+  description: "Fuzzy search pages by title, description, route.",
+  inputSchema,
+  cache: "5m",
+  async handler({ query, limit }) {
+    const pages = await getPagesList();
+    const fuse = new Fuse(pages, {
+      keys: ["title", "description", "route"],
+      threshold: 0.4,
+      includeScore: true
+    });
+    const results = fuse.search(query, { limit });
+    const items = results.map((r) => ({ ...r.item, score: r.score ?? 0 }));
+    return { content: [{ type: "text", text: JSON.stringify(items) }] };
+  }
+};
+export default tool;

package/dist/runtime/server/middleware/markdown.js

@@ -0,0 +1,62 @@
+import { withSiteUrl } from "#site-config/server/composables/utils";
+import { createError, defineEventHandler, setHeader } from "h3";
+import { useRuntimeConfig } from "nitropack/runtime";
+import { logger } from "../logger.js";
+import { convertHtmlToMarkdown, getMarkdownRenderInfo } from "../utils.js";
+export default defineEventHandler(async (event) => {
+  const renderInfo = getMarkdownRenderInfo(event);
+  if (!renderInfo)
+    return;
+  const { path, isExplicit } = renderInfo;
+  const config = useRuntimeConfig(event)["nuxt-ai-ready"];
+  const response = await event.fetch(path).catch((e) => {
+    logger.error(`Failed to fetch HTML for ${path}`, e);
+    return null;
+  });
+  if (!response) {
+    if (isExplicit) {
+      return createError({
+        statusCode: 500,
+        statusMessage: "Internal Server Error",
+        message: `Failed to fetch HTML for ${path}`
+      });
+    }
+    return;
+  }
+  if (!response.ok) {
+    if (isExplicit) {
+      return createError({
+        statusCode: response.status,
+        statusMessage: response.statusText,
+        message: `Failed to fetch HTML for ${path}`
+      });
+    }
+    return;
+  }
+  const contentType = response.headers.get("content-type") || "";
+  if (!contentType.includes("text/html")) {
+    if (isExplicit) {
+      return createError({
+        statusCode: 415,
+        statusMessage: "Unsupported Media Type",
+        message: `Expected text/html but got ${contentType} for ${path}`
+      });
+    }
+    return;
+  }
+  const html = await response.text();
+  const result = await convertHtmlToMarkdown(
+    html,
+    withSiteUrl(event, path),
+    config,
+    path,
+    event
+  );
+  setHeader(event, "content-type", "text/markdown; charset=utf-8");
+  if (config.markdownCacheHeaders) {
+    const { maxAge, swr } = config.markdownCacheHeaders;
+    const cacheControl = swr ? `public, max-age=${maxAge}, stale-while-revalidate=${maxAge}` : `public, max-age=${maxAge}`;
+    setHeader(event, "cache-control", cacheControl);
+  }
+  return result.markdown;
+});

package/dist/runtime/server/middleware/markdown.prerender.d.ts

@@ -0,0 +1,2 @@
+declare const _default: import("h3").EventHandler<import("h3").EventHandlerRequest, Promise<string | import("h3").H3Error<unknown> | undefined>>;
+export default _default;

package/dist/runtime/server/middleware/markdown.prerender.js

@@ -0,0 +1,35 @@
+import { withSiteUrl } from "#site-config/server/composables/utils";
+import { createError, defineEventHandler } from "h3";
+import { useRuntimeConfig } from "nitropack/runtime";
+import { convertHtmlToMarkdownMeta, getMarkdownRenderInfo } from "../utils.js";
+export default defineEventHandler(async (event) => {
+  if (!import.meta.prerender) {
+    return;
+  }
+  const renderInfo = getMarkdownRenderInfo(event, true);
+  if (!renderInfo)
+    return;
+  const { path } = renderInfo;
+  const runtimeConfig = useRuntimeConfig(event)["nuxt-ai-ready"];
+  const response = await event.fetch(path);
+  if (!response.ok) {
+    return createError({
+      statusCode: response.status,
+      statusMessage: response.statusText,
+      message: `Failed to fetch HTML for ${path}`
+    });
+  }
+  const html = await response.text();
+  if (html.includes("__NUXT_ERROR__") || html.includes("nuxt-error-page")) {
+    return createError({
+      statusCode: 404,
+      message: `Page rendered as error: ${path}`
+    });
+  }
+  const result = convertHtmlToMarkdownMeta(
+    html,
+    withSiteUrl(event, path),
+    runtimeConfig.mdreamOptions
+  );
+  return JSON.stringify(result);
+});

package/dist/runtime/server/routes/llms-full.txt.get.d.ts

@@ -0,0 +1,2 @@
+declare const _default: import("h3").EventHandler<import("h3").EventHandlerRequest, string>;
+export default _default;

package/dist/runtime/server/routes/llms-full.txt.get.js

@@ -0,0 +1,5 @@
+import { eventHandler, setHeader } from "h3";
+export default eventHandler((event) => {
+  setHeader(event, "Content-Type", "text/plain; charset=utf-8");
+  return "# llms-full.txt\n\nThis file is only available for prerendered routes.\nRun `nuxi generate` to generate this file.";
+});

package/dist/runtime/server/routes/llms.txt.get.js

@@ -1,23 +1,23 @@
-import { getSiteConfig } from "#site-config/server/composables/getSiteConfig";
 import { eventHandler, setHeader } from "h3";
-import { useRuntimeConfig } from "nitropack/runtime";
-import { normalizeLlmsTxtConfig } from "../../llms-txt.js";
+import { defineCachedFunction, useRuntimeConfig } from "nitropack/runtime";
+import { buildLlmsTxt } from "../../llms-txt-utils.js";
+const buildLlmsTxtCached = defineCachedFunction(
+  buildLlmsTxt,
+  {
+    name: "llms-txt",
+    group: "ai-ready",
+    maxAge: 60 * 10,
+    // 10 minutes
+    swr: true
+  }
+);
 export default eventHandler(async (event) => {
   const runtimeConfig = useRuntimeConfig(event)["nuxt-ai-ready"];
-  const siteConfig = getSiteConfig(event);
-  const llmsTxtConfig = runtimeConfig.llmsTxt;
-  const parts = [];
-  parts.push(`# ${siteConfig.name || siteConfig.url}`);
-  if (siteConfig.description) {
-    parts.push(`
-> ${siteConfig.description}
-`);
-  }
-  parts.push("<!-- Pages will be generated at build time -->\n");
-  const normalizedContent = normalizeLlmsTxtConfig(llmsTxtConfig);
-  if (normalizedContent) {
-    parts.push(normalizedContent);
-  }
+  const cacheEnabled = !import.meta.dev && runtimeConfig.cacheMaxAgeSeconds > 0;
+  const content = cacheEnabled ? await buildLlmsTxtCached(event) : await buildLlmsTxt(event);
   setHeader(event, "Content-Type", "text/plain; charset=utf-8");
-  return parts.join("\n");
+  if (cacheEnabled) {
+    setHeader(event, "Cache-Control", `public, max-age=${runtimeConfig.cacheMaxAgeSeconds}, s-maxage=${runtimeConfig.cacheMaxAgeSeconds}, stale-while-revalidate=3600`);
+  }
+  return content;
 });

package/dist/runtime/server/utils/pageData.d.ts

@@ -0,0 +1,25 @@
+/** Page entry from virtual module */
+export interface PageEntry {
+    route: string;
+    title: string;
+    description: string;
+    headings: string;
+    updatedAt: string;
+}
+/** Page data from JSONL (includes markdown for llms-full.txt) */
+export interface PageData extends PageEntry {
+    markdown: string;
+}
+/** Read page data - returns page data indexed by route */
+export declare function getPages(): Promise<Map<string, PageEntry>>;
+/** Get all page data including markdown (prerender only) */
+export declare function readPrerenderedPageData(): Promise<Map<string, PageData>>;
+/** Page list item for MCP tools/resources */
+export interface PageListItem {
+    route: string;
+    title: string;
+    description: string;
+    headings?: string;
+}
+/** Get pages as flat list for MCP consumption */
+export declare function getPagesList(): Promise<PageListItem[]>;

package/dist/runtime/server/utils/pageData.js

@@ -0,0 +1,25 @@
+export async function getPages() {
+  if (import.meta.dev)
+    return /* @__PURE__ */ new Map();
+  if (import.meta.prerender) {
+    return readPrerenderedPageData();
+  }
+  const m = await import("#ai-ready-virtual/page-data.mjs");
+  return m.pages?.length ? new Map(m.pages.map((p) => [p.route, p])) : /* @__PURE__ */ new Map();
+}
+export async function readPrerenderedPageData() {
+  if (!import.meta.prerender)
+    return /* @__PURE__ */ new Map();
+  const { readPageDataFromFilesystem } = await import("#ai-ready-virtual/read-page-data.mjs");
+  const pages = await readPageDataFromFilesystem();
+  return pages?.length ? new Map(pages.map((p) => [p.route, p])) : /* @__PURE__ */ new Map();
+}
+export async function getPagesList() {
+  const pages = await getPages();
+  return Array.from(pages.values()).map((p) => ({
+    route: p.route,
+    title: p.title || p.route,
+    description: p.description || "",
+    headings: p.headings || void 0
+  }));
+}

package/dist/runtime/server/utils/sitemap.d.ts

@@ -0,0 +1,6 @@
+import type { H3Event } from 'h3';
+export interface SitemapUrl {
+    loc: string;
+    lastmod?: string;
+}
+export declare function fetchSitemapUrls(event: H3Event): Promise<SitemapUrl[]>;

package/dist/runtime/server/utils/sitemap.js

@@ -0,0 +1,25 @@
+import { parseSitemapXml } from "@nuxtjs/sitemap/utils";
+import { logger } from "../logger.js";
+export async function fetchSitemapUrls(event) {
+  const sitemapRes = await event.$fetch("/sitemap.xml", { responseType: "text" }).catch(() => null);
+  if (!sitemapRes) {
+    logger.warn("Sitemap not found at /sitemap.xml - llms.txt will have no pages listed");
+    return [];
+  }
+  const result = await parseSitemapXml(sitemapRes).catch((e) => {
+    logger.warn("Failed to parse sitemap.xml:", e);
+    return { urls: [] };
+  });
+  const urls = result?.urls || [];
+  if (urls.length === 0) {
+    logger.warn("Sitemap is empty - llms.txt will have no pages listed");
+  }
+  return urls.map((entry) => {
+    if (typeof entry === "string")
+      return { loc: entry };
+    return {
+      loc: entry.loc,
+      lastmod: entry.lastmod instanceof Date ? entry.lastmod.toISOString() : entry.lastmod
+    };
+  });
+}

package/dist/runtime/server/utils.d.ts

@@ -1,9 +1,21 @@
+import type { H3Event } from 'h3';
 import type { ModulePublicRuntimeConfig } from '../../module.js';
-export declare function convertHtmlToMarkdownChunks(html: string, url: string, mdreamOptions: ModulePublicRuntimeConfig['mdreamOptions']): {
+export declare function normalizeWhitespace(text: string): string;
+export declare function getMarkdownRenderInfo(event: H3Event, explicitOnly?: boolean): {
+    path: string;
+    isExplicit: boolean;
+} | null;
+export declare function clientPrefersMarkdown(event: H3Event): boolean;
+export declare function convertHtmlToMarkdown(html: string, url: string, config: ModulePublicRuntimeConfig, route: string, event: H3Event): Promise<{
+    markdown: string;
+    title: string;
+    description: string;
+    headings: Record<string, string>[];
+}>;
+export declare function convertHtmlToMarkdownMeta(html: string, url: string, mdreamOptions: ModulePublicRuntimeConfig['mdreamOptions']): {
     updatedAt?: string | undefined;
     markdown: string;
-    chunks: import("mdream").MarkdownChunk[];
     title: string;
     description: string;
-    headings: Record<string, string[]>;
+    headings: Record<string, string>[];
 };

package/dist/runtime/server/utils.js

@@ -1,23 +1,96 @@
-import { htmlToMarkdown, TagIdMap } from "mdream";
+import { getHeader } from "h3";
+import { htmlToMarkdown } from "mdream";
 import { extractionPlugin } from "mdream/plugins";
 import { withMinimalPreset } from "mdream/preset/minimal";
-import { htmlToMarkdownSplitChunks } from "mdream/splitter";
-import { estimateTokenCount } from "tokenx";
-function stripFrontmatter(text) {
-  if (!text.startsWith("---\n"))
-    return text;
-  const endIdx = text.indexOf("\n---", 4);
-  if (endIdx === -1)
-    return text;
-  return text.slice(endIdx + 4).trimStart();
-}
-function normalizeWhitespace(text) {
+import { useNitroApp } from "nitropack/runtime";
+export function normalizeWhitespace(text) {
   return text.replace(/\u00A0/g, " ");
 }
-export function convertHtmlToMarkdownChunks(html, url, mdreamOptions) {
+export function getMarkdownRenderInfo(event, explicitOnly = false) {
+  const originalPath = event.path;
+  if (originalPath.startsWith("/api") || originalPath.startsWith("/_") || originalPath.startsWith("/@")) {
+    return null;
+  }
+  const isExplicit = originalPath.endsWith(".md");
+  if (explicitOnly && !isExplicit) {
+    return null;
+  }
+  const lastSegment = originalPath.split("/").pop() || "";
+  const hasExtension = lastSegment.includes(".");
+  const extension = hasExtension ? lastSegment.substring(lastSegment.lastIndexOf(".")) : "";
+  if (hasExtension && extension !== ".md") {
+    return null;
+  }
+  const isImplicit = !explicitOnly && clientPrefersMarkdown(event);
+  if (!isExplicit && !isImplicit) {
+    return null;
+  }
+  let path = isExplicit ? originalPath.slice(0, -3) : originalPath;
+  if (path === "/index") {
+    path = "/";
+  }
+  return { path, isExplicit };
+}
+export function clientPrefersMarkdown(event) {
+  const accept = getHeader(event, "accept") || "";
+  const secFetchDest = getHeader(event, "sec-fetch-dest") || "";
+  if (secFetchDest === "document") {
+    return false;
+  }
+  if (accept.includes("text/html")) {
+    return false;
+  }
+  return accept.includes("*/*") || accept.includes("text/markdown");
+}
+export async function convertHtmlToMarkdown(html, url, config, route, event) {
+  const nitroApp = useNitroApp();
+  let title = "";
+  let description = "";
+  const headings = [];
+  const extractPlugin = extractionPlugin({
+    title(el) {
+      title = el.textContent;
+    },
+    'meta[name="description"]': (el) => {
+      description = el.attributes.content || "";
+    },
+    "h1, h2, h3, h4, h5, h6": (el) => {
+      const text = el.textContent?.trim();
+      const level = el.name.toLowerCase();
+      if (text)
+        headings.push({ [level]: text });
+    }
+  });
+  let options = {
+    origin: url,
+    ...config.mdreamOptions
+  };
+  if (config.mdreamOptions?.preset === "minimal") {
+    options = withMinimalPreset(options);
+    options.plugins = [extractPlugin, ...options.plugins || []];
+  } else {
+    options.plugins = [extractPlugin, ...options.plugins || []];
+  }
+  await nitroApp.hooks.callHook("ai-ready:mdreamConfig", options);
+  let markdown = htmlToMarkdown(html, options);
+  const context = {
+    html,
+    markdown,
+    route,
+    title,
+    description,
+    isPrerender: false,
+    event
+  };
+  await nitroApp.hooks.callHook("ai-ready:markdown", context);
+  markdown = normalizeWhitespace(context.markdown);
+  return { markdown, title: normalizeWhitespace(title), description: normalizeWhitespace(description), headings };
+}
+export function convertHtmlToMarkdownMeta(html, url, mdreamOptions) {
   let title = "";
   let description = "";
   let updatedAt;
+  const headings = [];
   const extractPlugin = extractionPlugin({
     title(el) {
       title = el.textContent;
@@ -29,6 +102,12 @@ export function convertHtmlToMarkdownChunks(html, url, mdreamOptions) {
       if (!updatedAt && el.attributes.content) {
         updatedAt = el.attributes.content;
       }
+    },
+    "h1, h2, h3, h4, h5, h6": (el) => {
+      const text = el.textContent?.trim();
+      const level = el.name.toLowerCase();
+      if (text)
+        headings.push({ [level]: text });
     }
   });
   let options = {
@@ -42,40 +121,9 @@ export function convertHtmlToMarkdownChunks(html, url, mdreamOptions) {
     options.plugins = [extractPlugin, ...options.plugins || []];
   }
   const rawMarkdown = htmlToMarkdown(html, options);
-  const markdown = normalizeWhitespace(stripFrontmatter(rawMarkdown));
-  const rawChunks = htmlToMarkdownSplitChunks(html, {
-    ...options,
-    headersToSplitOn: [TagIdMap.h1, TagIdMap.h2, TagIdMap.h3],
-    origin: url,
-    chunkSize: 512,
-    stripHeaders: false,
-    lengthFunction(text) {
-      return estimateTokenCount(text);
-    }
-  });
-  const chunks = rawChunks.filter((chunk, idx) => {
-    chunk.content = normalizeWhitespace(chunk.content);
-    if (idx === 0 && chunk.content.startsWith("---\n")) {
-      const endIdx = chunk.content.indexOf("\n---", 4);
-      if (endIdx !== -1) {
-        chunk.content = chunk.content.slice(endIdx + 4).trimStart();
-        return chunk.content.length > 0;
-      }
-    }
-    return true;
-  });
-  const headings = chunks.reduce((set, chunk) => {
-    Object.entries(chunk.metadata?.headers || {}).forEach(([k, v]) => {
-      if (!set[k])
-        set[k] = [];
-      if (v && !set[k].includes(v))
-        set[k].push(v);
-    });
-    return set;
-  }, {});
+  const markdown = normalizeWhitespace(rawMarkdown);
   return {
     markdown,
-    chunks,
     title: normalizeWhitespace(title),
     description: normalizeWhitespace(description),
     headings,